Indy 10.6.2 idFTP in Delphi 7 - problem with native symbols in filenames while receiving files - probably bad behavior of DefStringEncoding

Question

I'm using Indy 10.6.2 together with Delphi 7 and Windows 10 64-bit with Polish language.

I run an FTP server: FTPServer: TIdFTPServer, and by using FTPClient: TIdFTP I am trying to get a file containing national symbols within its file name. Unfortunately, after calling the Get() function, on the server side in the OnRetrieveFile event, instead of national symbols in AFileName I end up with question marks, and this obviously leads to other exceptions.

For tests, I run both: server and client on the same machine and in the same application to eliminate any other disturbances.

On the client side, I already tried:

 FTPClient.DefStringEncoding := IndyTextEncoding_UTF8;
 FTPClient.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
 FTPClient.IOHandler.DefAnsiEncoding   := IndyTextEncoding_UTF8;

On the server side, I tried:

 ASender.Connection.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
 ASender.Connection.IOHandler.DefAnsiEncoding   := IndyTextEncoding_UTF8;

None of them makes any difference. I also tried other encodings, but also failed.

Below is my test application, together with text DFM values of client and server.

unit Glowny_FTP;

interface

uses
  Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
  Dialogs, StdCtrls, IdBaseComponent, IdComponent, IdTCPConnection,
  IdTCPClient, IdExplicitTLSClientServerBase, IdFTP, ZLibCompression, IdGlobal,
  IdIOHandler, IdIOHandlerStream, IdIOHandlerSocket, IdIOHandlerStack,
  IdCustomTCPServer, IdTCPServer, IdCmdTCPServer, IdFTPServer, IdContext,
  IdAntiFreezeBase, IdAntiFreeze;

type
  TForm1 = class(TForm)
    Button1: TButton;
    FTPClient: TIdFTP;
    FTPServer: TIdFTPServer;
    IdAntiFreeze1: TIdAntiFreeze;
    procedure Button1Click(Sender: TObject);
    procedure FTPServerUserLogin(ASender: TIdFTPServerContext;
      const AUsername, APassword: String; var AAuthenticated: Boolean);
    procedure FTPServerRetrieveFile(ASender: TIdFTPServerContext;
      const AFileName: WideString; var VStream: TStream);
  private
    { Private declarations }
  public
    { Public declarations }
  end;

var
  Form1: TForm1;

implementation

{$R *.dfm}

procedure TForm1.Button1Click(Sender: TObject);

begin

  FTPServer.DefaultPort    := 1350;
  FTPServer.Active         := True;

  FTPClient.Host     := '127.0.0.1';
  FTPClient.Port     := 1350;
  FTPClient.Username := 'new';
  FTPClient.Password := 'pass';
  FTPClient.Connect;
  // FTPClient.DefStringEncoding := IndyTextEncoding_UTF8;
  // FTPClient.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
  // FTPClient.IOHandler.DefAnsiEncoding   := IndyTextEncoding_UTF8;

  FTPClient.Get('sample ąęśćłó','C:\węzły.cpy',True,False);   // <-- filename containing national symbols
  FTPClient.Disconnect;
end;

procedure TForm1.FTPServerUserLogin(ASender: TIdFTPServerContext;
  const AUsername, APassword: String; var AAuthenticated: Boolean);
begin
  if (APassword='pass') then
    begin
      AAuthenticated := True;
    end else AAuthenticated := False;
end;

procedure TForm1.FTPServerRetrieveFile(ASender: TIdFTPServerContext;
  const AFileName: WideString; var VStream: TStream);
begin
 // ASender.Connection.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
 // ASender.Connection.IOHandler.DefAnsiEncoding   := IndyTextEncoding_UTF8;
  VStream := TFileStream.Create(AFileName+'.serv',fmOpenRead); //<-- here I get: "/sample ??????" without UTF8 and "/sample ????" with UTF8
end;

end.

and maybe interesting DFM part:

object FTPClient: TIdFTP
    Passive = True
    ConnectTimeout = 0
    DataPort = 20
    Password = 'TaJnEH1aS2loD3oSt4ePu'
    TransferType = ftBinary
    NATKeepAlive.UseKeepAlive = False
    NATKeepAlive.IdleTimeMS = 0
    NATKeepAlive.IntervalMS = 0
    ProxySettings.ProxyType = fpcmNone
    ProxySettings.Port = 0
    Left = 104
    Top = 72
  end
  object FTPServer: TIdFTPServer
    Bindings = <>
    DefaultPort = 1350
    TerminateWaitTime = 100
    CommandHandlers = <>
    ExceptionReply.Code = '500'
    ExceptionReply.Text.Strings = (
      'Unknown Internal Error')
    Greeting.Code = '220'
    Greeting.Text.Strings = (
      'Indy FTP Server ready.')
    MaxConnectionReply.Code = '300'
    MaxConnectionReply.Text.Strings = (
      'Too many connections. Try again later.')
    ReplyTexts = <>
    ReplyUnknownCommand.Code = '500'
    ReplyUnknownCommand.Text.Strings = (
      'Unknown Command')
    PathProcessing = ftppUnix
    AnonymousAccounts.Strings = (
      'anonymous'
      'ftp'
      'guest')
    OnUserLogin = FTPServerUserLogin
    OnRetrieveFile = FTPServerRetrieveFile
    SITECommands = <>
    MLSDFacts = []
    ReplyUnknownSITCommand.Code = '500'
    ReplyUnknownSITCommand.Text.Strings = (
      'Invalid SITE command.')
    Left = 104
    Top = 8
  end

---

Update:

After Remy's help still no efect. After UTF8Encode() inside FTPClient.Get() I got more question marks on server side. Now my code looks like this: (I check AFileName on Form1.Caption just for quick debug)

...
FTPClient.Connect;

FTPClient.DefStringEncoding := IndyTextEncoding_UTF8;
FTPClient.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
FTPClient.IOHandler.DefAnsiEncoding   := IndyTextEncoding_8Bit;

FTPClient.Get(UTF8Encode('sample ąęśćłó'),'C:\węzły.cpy',True,False);
FTPClient.Disconnect;

procedure TForm1.FTPServerRetrieveFile(ASender: TIdFTPServerContext;
  const AFileName: WideString; var VStream: TStream);
begin
  Form1.Caption := AFileName;
  VStream := TFileStream.Create('C:\tymczas z węzłami obl.aqr',fmOpenRead);
  //  VStream := TFileStream.Create(AFileName+'.serv',fmOpenRead);

end;

procedure TForm1.FTPServerConnect(AContext: TIdContext);
begin
  AContext.Connection.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
  AContext.Connection.IOHandler.DefAnsiEncoding   := IndyTextEncoding_UTF8;
end;

Answer 1

Finally I gave up with messy TextEncoding under D7. Lost three days and I'm in the same place.

Long investigation led me to TIdASCIIEncoding.GetBytes() no matter what encoding type I choose. Here any Char higher than $007F (127) is turned into "?" which is obvious for ASCII but not for UTF8 and ANSI.

I will use longer road around the problem that should work as long as I will communicate just between my two applications. I know it is patching but at least works.

"Solution" :

Inside FTPClient.Get(), instead of source string containing national symbols I put string encoded by IdEncoderMIME.EncodeString() and on server side inside FTPServerRetrieveFile() I delete first char "/" then my string is decoded by IdDecoderMIME.DecodeString() and finally I get proper filename on the other side.

Client side:

S := IdEncoderMIME1.EncodeString('sample ąęśćłó',IndyTextEncoding_UTF8,IndyTextEncoding_UTF8);
FTPClient.Get(S,'C:\węzły.cpy',True,False);

Server side:

S := Copy(AFileName,2,Length(AFileName));
S := IdDecoderMIME1.DecodeString(S,IndyTextEncoding_UTF8,IndyTextEncoding_UTF8);

Ugly but works...

Answer 2

Background

The IOHandler's DefStringEncoding needs to be set to the byte encoding that you want to use when transmitting strings over the socket. Indy will send out a Unicode string by encoding it to bytes using DefStringEncoding. And conversely, Indy will read in a Unicode string by decoding the received bytes to Unicode using DefStringEncoding.

In Delphi 2007 and earlier, there is an extra step involved. The IOHandler's DefAnsiEncoding needs to be set to the encoding that you want to use with AnsiStrings. Indy will send out an AnsiString by first decoding it from ANSI to Unicode using DefAnsiEncoding and then sending that Unicode using DefStringEncoding. Conversely, Indy will read in an AnsiString by first reading in a Unicode string using DefStringEncoding and then encoding that Unicode to ANSI using DefAnsiEncoding.

By default, DefStringEncoding is IndyTextEncoding_ASCII (for historical reasons), and DefAnsiEncoding is IndyTextEncoding_OSDefault.

On the client side

TIdFTP will automatically switch the IOHandler's DefStringEncoding to IndyTextEncoding_UTF8 if it determines that the server supports UTF-8. Without UTF-8 extensions, the FTP protocol requires the use of ASCII instead. So, you really shouldn't be messing with the client's DefStringEncoding at all.

You were originally passing in the remote filename as a Polish-encoded AnsiString, so having DefAnsiEncoding set to IndyTextEncoding_OSDefault makes sense on an OS set to a Polish locale. With DefStringEncoding set to IndyTextEncoding_UTF8, proper UTF-8 should be transmitted to the server.

You later changed the client to pass in a UTF-8 encoded AnsiString instead, but you set DefAnsiEncoding to IndyTextEncoding_8Bit, so the AnsiString does not get converted to Unicode correctly, and thus the subsequent conversion to UTF-8 is malformed. DefAnsiEncoding needs to be set to IndyTextEncoding_UTF8 in that situation. I would suggest leaving the DefAnsiEncoding set to IndyTextEncoding_OSDefault and use OS locale encoded AnsiStrings, unless you have a compelling reason not to.

On the server side

TIdFTPServer will automatically switch an IOHandler's DefStringEncoding to IndyTextEncoding_UTF8 only if the client issues an OPTS UTF-8 <NLST> or OPTS UTF8 ON command (which TIdFTP does, but other clients might not). Note that this is NOT in accordance with RFC 2640, which Indy does not fully implement yet.

You are setting both DefStringEncoding and DefAnsiEncoding to IndyTextEncoding_UTF8, which is typically OK, for the most part. You will end up with UTF-8 encoded AnsiStrings in events that expose access to AnsiStrings.

However, the OnRetrieveFile event uses WideString for its AFileName parameter in your version of Delphi. After the server has read in the filename from the socket as an AnsiString, it gets passed as-is to the event handler, converting it to WideString using the RTL's own conversion logic, which uses the OS's locale by default. So, in your situation, where the AnsiString is UTF-8 encoded but the OS locale is Polish, you will end up with a malformed conversion. Fortunately, you can mitigate that issue by calling the RTL's SetMultiByteConversionCodePage() function beforehand to perform AnsiString<->WideString conversions using codepage 65001 (UTF-8) instead.

However, several of TIdFTPServer's events use WideString parameters in Delphi 2007 and earlier, and thus there are quite a few areas of TIdFTPServer's internals that rely on the RTL's default AnsiString<->WideString conversions. So, I would suggest leaving DefAnsiEncoding set to IndyTextEncoding_OSDefault on the server side as well, unless you have a compelling reason not to. If you want to force DefStringEncoding to IndyTextEncoding_UTF8 regardless of OPTS commands, that is OK (provided your clients only ever send UTF-8 encoded paths).

---

With that said, try this:

unit Glowny_FTP;

interface

uses
  Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
  Dialogs, StdCtrls, IdBaseComponent, IdComponent, IdTCPConnection,
  IdTCPClient, IdExplicitTLSClientServerBase, IdFTP, ZLibCompression, IdGlobal,
  IdIOHandler, IdIOHandlerStream, IdIOHandlerSocket, IdIOHandlerStack,
  IdCustomTCPServer, IdTCPServer, IdCmdTCPServer, IdFTPServer, IdContext,
  IdAntiFreezeBase, IdAntiFreeze;

type
  TForm1 = class(TForm)
    Button1: TButton;
    FTPClient: TIdFTP;
    FTPServer: TIdFTPServer;
    IdAntiFreeze1: TIdAntiFreeze;
    procedure Button1Click(Sender: TObject);
    procedure FTPServerConnect(ASender: TIdContext);
    procedure FTPServerUserLogin(ASender: TIdFTPServerContext;
      const AUsername, APassword: String; var AAuthenticated: Boolean);
    procedure FTPServerRetrieveFile(ASender: TIdFTPServerContext;
      const AFileName: WideString; var VStream: TStream);
  private
    { Private declarations }
  public
    { Public declarations }
  end;

var
  Form1: TForm1;

implementation

{$R *.dfm}

procedure TForm1.Button1Click(Sender: TObject);
begin
  FTPServer.DefaultPort    := 1350;
  FTPServer.Active         := True;

  FTPClient.Host     := '127.0.0.1';
  FTPClient.Port     := 1350;
  FTPClient.Username := 'new';
  FTPClient.Password := 'pass';
  FTPClient.Connect;
  try
    // FTPClient.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
    // FTPClient.IOHandler.DefAnsiEncoding := IndyTextEncoding_OSDefault;
    FTPClient.Get('sample ąęśćłó', 'C:\węzły.cpy', True, False);

    { or:
    //FTPClient.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
    FTPClient.IOHandler.DefAnsiEncoding := IndyTextEncoding_UTF8;
    FTPClient.Get(UTF8Encode('sample ąęśćłó'), 'C:\węzły.cpy', True, False);
    }
  finally
    FTPClient.Disconnect;
  end;
end;

procedure TForm1.FTPServerConnect(ASender: TIdContext);
begin
  ASender.Connection.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
  //ASender.Connection.IOHandler.DefAnsiEncoding := IndyTextEncoding_OSDefault;
end;

procedure TForm1.FTPServerUserLogin(ASender: TIdFTPServerContext;
  const AUsername, APassword: String; var AAuthenticated: Boolean);
begin
  AAuthenticated := (AUsername = 'new') and (APassword = 'pass');
end;

procedure TForm1.FTPServerRetrieveFile(ASender: TIdFTPServerContext;
  const AFileName: WideString; var VStream: TStream);
begin
  VStream := TFileStream.Create(AFileName + '.serv', fmOpenRead);
end;

end.

Or this:

unit Glowny_FTP;

interface

uses
  Windows, Messages, SysUtils, Variants, Classes, Graphics, Controls, Forms,
  Dialogs, StdCtrls, IdBaseComponent, IdComponent, IdTCPConnection,
  IdTCPClient, IdExplicitTLSClientServerBase, IdFTP, ZLibCompression, IdGlobal,
  IdIOHandler, IdIOHandlerStream, IdIOHandlerSocket, IdIOHandlerStack,
  IdCustomTCPServer, IdTCPServer, IdCmdTCPServer, IdFTPServer, IdContext,
  IdAntiFreezeBase, IdAntiFreeze;

type
  TForm1 = class(TForm)
    Button1: TButton;
    FTPClient: TIdFTP;
    FTPServer: TIdFTPServer;
    IdAntiFreeze1: TIdAntiFreeze;
    procedure FormCreate(Sender: TObject);
    procedure Button1Click(Sender: TObject);
    procedure FTPServerConnect(ASender: TIdContext);
    procedure FTPServerUserLogin(ASender: TIdFTPServerContext;
      const AUsername, APassword: String; var AAuthenticated: Boolean);
    procedure FTPServerRetrieveFile(ASender: TIdFTPServerContext;
      const AFileName: WideString; var VStream: TStream);
  private
    { Private declarations }
  public
    { Public declarations }
  end;

var
  Form1: TForm1;

implementation

{$R *.dfm}

procedure TForm1.FormCreate(Sender: TObject);
begin
  SetMultiByteConversionCodePage(CP_UTF8);
end;

procedure TForm1.Button1Click(Sender: TObject);
begin
  FTPServer.DefaultPort    := 1350;
  FTPServer.Active         := True;

  FTPClient.Host     := '127.0.0.1';
  FTPClient.Port     := 1350;
  FTPClient.Username := 'new';
  FTPClient.Password := 'pass';
  FTPClient.Connect;
  try
    // FTPClient.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
    // FTPClient.IOHandler.DefAnsiEncoding := IndyTextEncoding_OSDefault;
    FTPClient.Get('sample ąęśćłó', 'C:\węzły.cpy', True, False);

    { or:
    //FTPClient.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
    FTPClient.IOHandler.DefAnsiEncoding := IndyTextEncoding_UTF8;
    FTPClient.Get(UTF8Encode('sample ąęśćłó'), 'C:\węzły.cpy', True, False);
    }
  finally
    FTPClient.Disconnect;
  end;
end;

procedure TForm1.FTPServerConnect(ASender: TIdContext);
begin
  ASender.Connection.IOHandler.DefStringEncoding := IndyTextEncoding_UTF8;
  ASender.Connection.IOHandler.DefAnsiEncoding := IndyTextEncoding_UTF8;
end;

procedure TForm1.FTPServerUserLogin(ASender: TIdFTPServerContext;
  const AUsername, APassword: String; var AAuthenticated: Boolean);
begin
  AAuthenticated := (AUsername = 'new') and (APassword = 'pass');
end;

procedure TForm1.FTPServerRetrieveFile(ASender: TIdFTPServerContext;
  const AFileName: WideString; var VStream: TStream);
begin
  VStream := TFileStream.Create(AFileName + '.serv', fmOpenRead);
end;

end.