How to get all the subclasses of an abstract class in different files

Question

I'm trying to apply design pattern to make it extendable and easy to maintain.

Here is my file structure:

/evaluation
│── metrics/
│──│── metric.py  # abstract class 
│──│── QWK.py
│──│── PCC.py
│── evaluators/ 
│──│── __init__.py
│──│── evaluator.py # abstract class
│──│── score_evaluator.py
│──│── feedback_evaluator.py
│── evaluator_factory.py

evaluator.py

class Evaluator(ABC):
    def __init__(self, evaluation_config, framework_config, **kwargs):

        self.evaluation_config = evaluation_config
        self.framework_config = framework_config

        self.logger = WandbLogger()

    @abstractmethod
    def load_evaluation_data(self, data_path):
        pass

    @abstractmethod
    def __call__(self, *args, **kwargs):
        pass

score_evaluator.py

class ScoreEvaluator(Evaluator):
    def __init__(self, evaluation_config, framework_config, **kwargs):
        super().__init__(evaluation_config, framework_config, **kwargs)

        self.score_generator = ScoreGenerator(llm=LLMClient(),
                                              vlm=VLMClient(),
                                              config=self.framework_config)


    def load_evaluation_data(self, data_path):
        ...
        return evaluation_data


    async def __call__(self, *args, **kwargs):
        ... 

        return evaluation_results

evaluator_factory.py

from evaluation.evaluators.evaluator import Evaluator

class EvaluatorFactory:
    def __init__(self, framework_config, evaluation_config, **kwargs):
        self.framework_config = framework_config
        self.evaluation_config = evaluation_config
        self.kwargs = kwargs

        self.evaluators = self._create_evaluators()

    def _create_evaluators(self):
        evaluators = []
        evaluator_classes = Evaluator.__subclasses__()
        print(evaluator_classes) # bug here, get empty list 
        evaluator_module = importlib.import_module("evaluation.evaluators.evaluator")
        print(evaluator_module) # second attempt using importlib, and inspect still get empty dict. 
        evaluators = {
            name: cls()
            for name, cls in inspect.getmembers(evaluator_module, inspect.isclass)
            if issubclass(cls, evaluator_module.Evaluator) and cls is not evaluator_module.Evaluator
        }

        print(evaluators)
        for cls in evaluator_classes:
            evaluators.append(cls(self.framework_config, self.evaluation_config, **self.kwargs))

        return evaluators

    async def evaluate(self):
        evaluation_results = {}
        print(len(self.evaluators))
        for evaluator in self.evaluators:
            results = await evaluator()
            evaluation_results[evaluator.__name__] = results

        return evaluation_results

Dear design pattern specialists:), help me.

I also want to design something with the metrics:

• As some metrics are only designed for specific evaluator, so I don't know how is the best practice to design this, for example when someone in the team try to create a new metric for an evaluator, it's easy to scale without modify the code.

Thank youuuuuu so much~

---

Edit 1

I figured out that __subclasses__ only works when the subclasses already loaded in memory, to do so I need to register it in the __init__ file:

evaluators/__init__.py

from evaluation.evaluators.score_evaluator import ScoreEvaluator
from evaluation.evaluators.feedback_evaluator import FeedbackEvaluator

__all__ = ["ScoreEvaluator", "FeedbackEvaluator"]

Still waiting for any other approaches and my second concern 😊