PDF Scraping with Templated Document

Question

I cannot scrape other details from PDF File. Some document scraped all, while others are not. And this is the following issue I am encountering.

I am scraping a Sample PDF File.

CASE1: Definition and Class Characteristic should be in one excel cell under Job Description column, BUT. Sometimes, it could be Definition and Class Characteristics or Definition and Distinguishing Characteristics or it could be `Definition and Class Characteristic AND Distinguishing Characteristics

CASE2: Python is case sensitive, other document only uses Distinguishing CharacteristicS and other Job Description file does not haveS in the Characteristic word. How can we handle that?

CASE3: EXAMPLE OF DUTIES should also be in one excel cell.

CASE4: Education and Experience column should be a combination of Knowledge of and Ability to and sometimes Education and Training

Please see my code below:I hope any once could help me on this.

import fitz  # PyMuPDF
import pandas as pd
import re
import os


# Step 1: Extract Text from PDF
def extract_text_from_pdf(pdf_path):
    document = fitz.open(pdf_path)
    pdf_text = ""

    for page_num in range(len(document)):
        page = document.load_page(page_num)
        pdf_text += page.get_text()

    return pdf_text


# Define regular expression patterns with case insensitivity and optional character matching
job_title_re = re.compile(r'UNIT:\s*(.*?)\s*Class Specification', re.DOTALL | re.IGNORECASE)
class_specification_re = re.compile(r'Class Specification\s*(.*?)\s*DEFINITION\s*(.*?)\s', re.DOTALL | re.IGNORECASE)
definition_re = re.compile(r'DEFINITION\s*(.*?)\s*CLASS CHARACTERISTICS', re.DOTALL | re.IGNORECASE)
definition1_re = re.compile(r'DEFINITION\s*(.*?)\s*DISTINGUISHING CHARACTERISTICS', re.DOTALL | re.IGNORECASE)
class_characteristics_re = re.compile(r'CLASS CHARACTERISTICS\s*(.*?)\s*EXAMPLE OF DUTIES', re.DOTALL | re.IGNORECASE)
distinguishing_characteristics_re = re.compile(r'DISTINGUISHING CHARACTERISTICS\s*(.*?)\s*EXAMPLE OF DUTIES', re.DOTALL | re.IGNORECASE)
example_of_duties_re = re.compile(r'EXAMPLE OF DUTIES\s*(.*?)\s*MINIMUM QUALIFICATIONS', re.DOTALL | re.IGNORECASE)
knowledge_of_re = re.compile(r'Knowledge of:\s*(.*?)\s*Ability to:', re.DOTALL | re.IGNORECASE)
ability_to_re = re.compile(r'Ability to:\s*(.*?)\s*Experience and Training', re.DOTALL | re.IGNORECASE)
experience_and_training_re = re.compile(r'Experience and Training\s*(.*?)\s*WORKING CONDITIONS', re.DOTALL | re.IGNORECASE)


def extract_section(text, pattern):
    match = pattern.search(text)
    return match.group(1).strip() if match else ""


def process_pdfs_in_folder(pdf_folder, excel_path):
    all_data = []

    for pdf_file in os.listdir(pdf_folder):
        if pdf_file.endswith('.pdf'):
            pdf_path = os.path.join(pdf_folder, pdf_file)
            pdf_text = extract_text_from_pdf(pdf_path)

            # Extract sections using regex patterns
            job_title = extract_section(pdf_text, job_title_re)
            class_specification = extract_section(pdf_text, class_specification_re)
            definition = extract_section(pdf_text, definition_re)
            definition1 = extract_section(pdf_text, definition1_re)
            class_characteristics = extract_section(pdf_text, class_characteristics_re)
            distinguishing_characteristics = extract_section(pdf_text, distinguishing_characteristics_re)
            example_of_duties = extract_section(pdf_text, example_of_duties_re)
            knowledge_of = extract_section(pdf_text, knowledge_of_re)
            ability_to = extract_section(pdf_text, ability_to_re)
            experience_and_training = extract_section(pdf_text, experience_and_training_re)

            # Prepare data for the DataFrame
            data = {
                "Job Title": job_title + "\n" + class_specification,
                "Definition and Class Characteristics": definition + "\n" + definition1 + "\n" + class_characteristics + "\n" + distinguishing_characteristics,
                "Example of Duties": example_of_duties,
                "Knowledge, Ability, Experience, and Training": knowledge_of + "\n" + ability_to + "\n" + experience_and_training
            }

            all_data.append(data)

    # Create DataFrame from all data
    new_df = pd.DataFrame(all_data)

    # Check if the Excel file exists
    if os.path.exists(excel_path):
        existing_df = pd.read_excel(excel_path)
        final_df = pd.concat([existing_df, new_df], ignore_index=True)
    else:
        final_df = new_df

    # Write to Excel file
    final_df.to_excel(excel_path, index=False)
    print(f"Data has been written to {excel_path}")


# Define the paths
pdf_folder = r'C:\Users\donna\PycharmProjects\scrapePdf\pythonProject\.venv\OceansideCA\JobTitle'
excel_path = r'C:\Users\donna\PycharmProjects\scrapePdf\pythonProject\.venv\OceansideCA\JD_Excel.xlsx'

# Run the script
process_pdfs_in_folder(pdf_folder, excel_path)