Pgf backend in matplotlib doesn't use the defined fonts

Question

Context: I'm producing a report with lot's of figures with some LateX content, and to maintain layout coherence I use a predefined mplstyle file with the following related definitions that I need to keep across the report.

# FONT
font.family: serif
font.size: 9.0
font.serif: Palatino
font.sans-serif: DejaVu Sans
font.weight: normal
#font.stretch: normal

# LATEX 
text.usetex: True

Everything's working fine, but as I need to use siunitx in one figure, I started using pgf backend just for that. Here is a short version of the code:

import matplotlib.pyplot as plt
import pandas as pd
import matplotlib as mpl
import seaborn as sns

PGF_WITH_LATEX = {                      # setup matplotlib to use latex for output
    "pgf.texsystem": "pdflatex",        # change this if using xetex or lautex
    "text.usetex": True,                # use LaTeX to write all text
    "font.family": "serif",
    "font.size": 9.0,
    "font.serif": "Palatino",
    "font.sans-serif": "DejaVu Sans",
    "font.weight": "normal",
    "pgf.preamble": "\n".join([  # plots will use this preamble
        r"\usepackage[utf8]{inputenc}",
        r"\usepackage[T1]{fontenc}",
        r"\usepackage[detect-all,locale=UK]{siunitx}",
        r"\usepackage{amssymb}",
    ])
}


def get_binarized_categories(categories=['None', 'Low', 'Medium', 'Med +', 'High', 'Very High'],
                             bins=[-0.01, 0.000001, 0.10, 0.20, 0.30, 0.666, 1], type_eng='regular'):
    criteria_str = 'C'
    label_cats = []
    special_cats = ['None', 'Low', 'Very High']
    for i, cat in enumerate(categories):
        if cat not in special_cats:
            bin_low, bin_high = bins[i], bins[i+1]
            label = f'${bin_low:9.3f}\\leqslant\\Delta {criteria_str}<{bin_high:9.3f}$'
        elif cat == 'None':
            label = f'$\\Delta {criteria_str}=0$'
        elif cat == 'Low':
            bin_low, bin_high = bins[i], bins[i+1]
            if type_eng == 'regular':
                sc_notation = f'{bin_low:.0e}'
            elif type_eng == 'pgf':
                sc_notation = f'$\\num{{{bin_low}}}$'
            label = f'${sc_notation}\\leqslant\\Delta {criteria_str}<{bin_high:9.3f}$'
        elif cat == 'Very High':
            label = f'$\\Delta {criteria_str}\\geqslant{{{bins[-2]:9.3f}}}$'
        label_cats.append(label)
    return categories, bins, label_cats


df = pd.DataFrame({'diff': [0, 0.00001, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]})

# Block ORIGINAL: The original figures (which keep the right fonts)
sns.set_theme(style="whitegrid")
plt.style.use('./python.mplstyle')
mpl.rcParams['text.latex.preamble'] = [r'\usepackage{amssymb}']
palette = sns.color_palette("deep", n_colors=11)
sns.set_palette(palette, n_colors=11)
fig, ax = plt.subplots(2, 3, constrained_layout=True)
categories, bins, label_cats = get_binarized_categories(type_eng='regular')
df = df.assign(slope=pd.cut(df['diff'], bins=bins, precision=6, labels=categories))
for id, (ax, category) in enumerate(zip(fig.axes, categories)):
    up_str = f'{label_cats[id]}'
    ax.text(.5, 1.01, up_str, horizontalalignment='center', transform=ax.transAxes, fontsize='small')
fig.savefig('test_latex_regular.pdf', bbox_inches='tight')

# Block PGF: with 'pgf' backend (that does not keep the right fonts)
sns.set_theme(style="whitegrid")
plt.style.use('./python.mplstyle')
palette = sns.color_palette("deep", n_colors=11)
sns.set_palette(palette, n_colors=11)
mpl.use('pgf')
mpl.rcParams.update(PGF_WITH_LATEX)
fig, ax = plt.subplots(2, 3, constrained_layout=True)
categories, bins, label_cats = get_binarized_categories(type_eng='pgf')
df = df.assign(slope=pd.cut(df['diff'], bins=bins, precision=6, labels=categories))
for id, (ax, category) in enumerate(zip(fig.axes, categories)):
    up_str = f'{label_cats[id]}'
    ax.text(.5, 1.01, up_str, horizontalalignment='center', transform=ax.transAxes, fontsize='small')
fig.savefig('test_latex_pgf.pdf', bbox_inches='tight')

This code produces this figure with the right fonts (and without pgf - produced by Block Original in the code)

However, when using pgfbackend (as produced by Block PGF in code), the fonts are incorrect.

So, my main question is how may I fix this problem with the fonts?

A secondary but connected issue that I noted was that I couldn't revert the backend to the one before choosing pgf (I don't know exactly what backend it is, so I assume it's the matplotlib default). Ie, if I produce the pgf figure first and then need to produce regular figures, the fonts keep being messed up.