Reputation: 428
With aioftp
in Python, I want to recursively list all files in an FTP folder.
What's the best way to do that? How can a recursive function be constructed to do that?
Here is my first attempt, which does not work:
import aioftp
import asyncio
async def async_list_all_files(
host: str, username: str, password: str, path: str = "/"
) -> list[str]:
async def list_files_recursive_parallel(client: aioftp.Client, path: str = "") -> list[str]:
task_queue: list[asyncio.Future] = []
file_list: list[str] = []
async for entry in client.list(path):
entry_full_path: str = entry[0].as_posix()
# Entry metadata (type hint added)
entry_metadata: dict[str, str | int] = entry[1]
# Handle files and directories
if entry_metadata["type"] == "file":
file_list.append(entry_full_path)
elif entry_metadata["type"] == "dir":
task_queue.append(
asyncio.ensure_future(list_files_recursive_parallel(client, entry_full_path))
)
else:
raise ValueError(
f"Unknown file type: {entry_metadata['type']} from entry='{entry}'"
)
# Process the task queue
if task_queue:
results = await asyncio.gather(*task_queue, return_exceptions=True)
for result in results:
if isinstance(result, list):
file_list.extend(result)
else:
logger.warning(f"Error during task: {result}")
return file_list
async def main() -> list[str]:
client = aioftp.Client()
await client.connect(host)
await client.login(username, password)
# Fetch all files recursively
return await list_files_recursive_parallel(client, path)
# Run the asynchronous `main` function and return results
return await main()
ftp_file_list = asyncio.run(
async_list_all_files(
host=creds["host"],
username=creds["username"],
password=creds["password"],
)
)
Upvotes: 0
Views: 22