Store files in different subdirectories using python

Question

I have a pandas dataframe df that looks like this:

import pandas as pd
d = {'user': ['Peter', 'Peter', 'Peter', 'Peter', 'David', 'David', 'David', 'Emma', 'Joyce', 'Joyce', 'Joyce'], 'date': ['2019-03-04', '2019-03-04', '2019-03-04', '2019-03-04', '2019-03-04', '2019-03-04', '2019-03-04', '2019-03-04', '2019-03-04', '2019-03-04', '2019-03-04'], 'lat': [37.749798119650926, 37.751028173710736, 37.751698332490214, 37.75180012822952, 38.122893081890844, 38.124108467926035, 38.12743379574882, 37.89363489791644, 37.53620628385582, 37.53804390907164, 37.54044296272588], 'lon': [-122.49230146408082, -122.49229073524474, -122.49170064926147, -122.48974800109862, -122.24205136299133, -122.23907947540283, -122.23867177963257, -122.07653760910033, -121.99707984924316, -121.99315309524536, -121.9914150238037]}
df = pd.DataFrame(data=d)
df

user    date        lat         lon
Peter   2019-03-04  37.749798   -122.492301
Peter   2019-03-04  37.751028   -122.492291
Peter   2019-03-04  37.751698   -122.491701
Peter   2019-03-04  37.751800   -122.489748
David   2019-03-04  38.122893   -122.242051
David   2019-03-04  38.124108   -122.239079
David   2019-03-04  38.127434   -122.238672
Emma    2019-03-04  37.893635   -122.076538
Joyce   2019-03-04  37.536206   -121.997080
Joyce   2019-03-04  37.538044   -121.993153
Joyce   2019-03-04  37.540443   -121.991415

Using the code below I'm able to create four separate folium maps grouped per user that show the coordinates on the map. The four map files are named after the user: Peter.html, David.html, Emma.html and Joyce.html.

import folium

users = list(df.user.unique())

def create_user_map(user):
    m = folium.Map(location=[37.733795, -122.446747], 
           zoom_start=9, 
           min_zoom=10, 
           max_zoom=19,
           control_scale=True)
 
    df_user = df[df.user==user]
    for row in df_user.itertuples():
         folium.CircleMarker( location=[row.lat, row.lon],
            radius=4,
            fill=True,
           fill_opacity=0.5).add_to(m)
    return m

for user in users:
    user_map = create_user_map(user)
    user_file = f"{user}.html"
    user_map.save(user_file)

Now I would like to automatically store these files in the corresponding subdirectories according to the folder structure below. How can I extend the loop above to achieve this?

Report/
└── Report_per_date/ 
    ├── 2019-03-01/
    ├── 2019-03-02/
    ├── 2019-03-03/
    └── 2019-03-04/
         └── Users/
             └── Peter/
                 └── Peter.html
             └── David/
                 └── David.html
             └── Emma/
                 └── Emma.html
             └── Joyce/
                 └── Joyce.html

I was hoping I could achieve this as follows:

import os
rootdir = pathlib.Path('./Report')
report_per_date = df.apply(lambda x: rootdir / 'Report_per_date' / x['date'] / 'Users' / x['user'] / f"{x['user']}.html", axis='columns')

for mapfile, data in df.groupby(report_per_date):
    mapfile.parent.mkdir(parents=True, exist_ok=True)
    user_map = create_user_map(user)
    user_map.save(mapfile)

Unfortunately this is returning an AttributeError: 'PosixPath' object has no attribute 'write'

Answer 1

You can change your loop to include dates, also create a 'path' column in your df.

df['path'] = df.apply(lambda x: rootdir / 'Report_per_date' / x['date'] / 'Users' / x['user'] / f"{x['user']}.html",
                      axis='columns')

dates = list(df.date.unique())


def create_user_map(df_user):
    m = folium.Map(location=[37.733795, -122.446747], 
               zoom_start=9, 
               min_zoom=10, 
               max_zoom=19,
               control_scale=True)
 
    for row in df_user.itertuples():
          folium.CircleMarker( location=[row.lat, row.lon],
                radius=4,
                fill=True,
                fill_opacity=0.5).add_to(m )
     
    return m

for user in users:
    for date in dates:
        data = df[(df.user==user)&(df.date==date)]
        path = data.iloc[0]['path']        
        path.parent.mkdir(parents=True, exist_ok=True)            
        user_map = create_user_map(data)
        user_map.save(str(path))