Text Processing with Regular Expressions

Cleaning text

import re


def clean(text):
    text = re.sub(r"<[^>]+>", "", text)          # strip simple markup
    text = re.sub(r"https?://\S+", "", text)     # strip URLs
    text = re.sub(r"[^\w\s.,!?-]", "", text)     # strip odd symbols
    text = re.sub(r"\s+", " ", text)             # collapse whitespace
    return text.strip()


messy = """
   <p>Visit   https://example.com/page?x=1 for   more!!</p>
   Contact:  meera@@@example.com   #urgent
"""

print(clean(messy))

Four substitutions applied in order. Doing it in one enormous pattern would be shorter and unreadable; a sequence of named steps can be tested and changed one at a time.

Normalising whitespace and case

import re


def normalise_name(raw):
    name = re.sub(r"\s+", " ", raw.strip())
    name = re.sub(r"\s*([-'])\s*", r"\1", name)      # tidy hyphens and apostrophes
    return re.sub(r"\b[a-z]", lambda m: m.group().upper(), name.lower())


for raw in ["  meera   NAIR ", "jean - luc  picard", "o'  brien"]:
    print(f"{raw!r:<24}-> {normalise_name(raw)!r}")

Extracting structured data

import re

INVOICE = re.compile(r"""
    ^INV-
    (?P<year>\d{4})-
    (?P<sequence>\d{5})
    \s+
    (?P<customer>[A-Za-z ]+?)      # lazy, so it stops before the amount
    \s+
    Rs\s*(?P<amount>[\d,]+\.\d{2})
    \s+
    (?P<status>PAID|DUE|OVERDUE)$
""", re.VERBOSE)

lines = [
    "INV-2026-00042 Meera Nair Rs 12,500.00 PAID",
    "INV-2026-00043 Arun Kumar Rs 8,750.50 DUE",
    "INV-2025-00988 Sara Iqbal Rs 1,200.00 OVERDUE",
    "malformed line",
]

records = []
for line in lines:
    match = INVOICE.match(line)
    if not match:
        print("skipped:", line)
        continue
    record = match.groupdict()
    record["amount"] = float(record["amount"].replace(",", ""))
    records.append(record)

total = sum(r["amount"] for r in records)
outstanding = sum(r["amount"] for r in records if r["status"] != "PAID")

for r in records:
    print(f"{r['customer']:<14}{r['amount']:>12,.2f}  {r['status']}")

print(f"{'TOTAL':<14}{total:>12,.2f}")
print(f"{'OUTSTANDING':<14}{outstanding:>12,.2f}")

Finding and counting

import re
from collections import Counter

text = """
Python is readable. Python is widely used.
python, PYTHON and Python appear here.
"""

print(len(re.findall(r"\bpython\b", text, re.IGNORECASE)))     # 5

words = re.findall(r"\b[a-z]+\b", text.lower())
print(Counter(words).most_common(3))

sentences = re.split(r"(?<=[.!?])\s+", text.strip())
print(len(sentences), "sentences")
for s in sentences:
    print(" -", s)

(?<=[.!?])\s+ splits after a sentence ending punctuation mark without consuming it. That is a lookbehind doing exactly what it is for.

Masking sensitive values

import re

RULES = [
    (re.compile(r"\b(\d{4})\d{8,10}(\d{4})\b"), r"\1********\2"),      # card numbers
    (re.compile(r"\b[\w.+-]+@([\w-]+\.[\w.]+)\b"), r"***@\1"),         # emails
    (re.compile(r"\b([6-9])\d{8}(\d)\b"), r"\1********\2"),            # phones
    (re.compile(r"(?i)(password\s*[=:]\s*)\S+"), r"\1********"),       # passwords
]


def redact(text):
    for pattern, replacement in RULES:
        text = pattern.sub(replacement, text)
    return text


log = """
user meera@example.com logged in
card 1234567812345678 charged
phone 9876543210 verified
password = hunter2
"""

print(redact(log))

Reformatting

import re


def to_snake_case(name):
    name = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", name)
    name = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", name)
    return name.replace("-", "_").replace(" ", "_").lower()


def to_camel_case(name):
    parts = re.split(r"[_\s-]+", name.strip())
    return parts[0].lower() + "".join(p.title() for p in parts[1:])


for name in ["HTTPResponseCode", "userFirstName", "my-variable name"]:
    print(f"{name:<22}{to_snake_case(name):<24}{to_camel_case(name)}")

import re


def slugify(title):
    slug = title.lower()
    slug = re.sub(r"[^\w\s-]", "", slug)      # drop punctuation
    slug = re.sub(r"[\s_]+", "-", slug)       # spaces to hyphens
    slug = re.sub(r"-+", "-", slug)           # collapse repeats
    return slug.strip("-")


for title in ["Python: Regex & Text Processing!", "  Hello -- World  "]:
    print(f"{title!r:<40}{slugify(title)}")

Working line by line

import re

SETTING = re.compile(r"^\s*(?P<key>[\w.]+)\s*[=:]\s*(?P<value>.*?)\s*(?:#.*)?$")
SECTION = re.compile(r"^\s*\[(?P<name>[^\]]+)\]\s*$")


def parse_config(text):
    result = {}
    section = result
    for number, line in enumerate(text.splitlines(), start=1):
        if not line.strip() or line.strip().startswith("#"):
            continue
        if (match := SECTION.match(line)):
            section = result.setdefault(match.group("name"), {})
            continue
        if (match := SETTING.match(line)):
            section[match.group("key")] = match.group("value")
            continue
        print(f"line {number} ignored: {line.strip()!r}")
    return result


config = """
# global settings
debug = true

[database]
host = localhost      # the server
port: 5432

[cache]
ttl = 300
nonsense line here
"""

import pprint
pprint.pprint(parse_config(config))

Search and replace across files

import re
from pathlib import Path


def replace_in_files(folder, pattern, replacement, glob="*.txt", dry_run=True):
    compiled = re.compile(pattern)
    for path in Path(folder).rglob(glob):
        original = path.read_text(encoding="utf-8")
        updated, count = compiled.subn(replacement, original)
        if count:
            print(f"{path}: {count} replacement(s)")
            if not dry_run:
                path.write_text(updated, encoding="utf-8")


# replace_in_files(".", r"\bcolour\b", "color", dry_run=True)

Note the dry_run default. Any script that edits files in bulk should show what it will do before it does it.

Validating a form

import re

FIELDS = {
    "username": (re.compile(r"[a-z][a-z0-9_]{2,19}"),
                 "3 to 20 characters, starting with a letter"),
    "pin":      (re.compile(r"\d{6}"), "exactly six digits"),
    "phone":    (re.compile(r"[6-9]\d{9}"), "ten digits starting 6 to 9"),
    "email":    (re.compile(r"[^@\s]+@[^@\s]+\.[a-z]{2,}"), "a valid address"),
    "date":     (re.compile(r"\d{4}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])"),
                 "YYYY-MM-DD"),
}


def validate(data):
    problems = {}
    for field, (pattern, message) in FIELDS.items():
        value = data.get(field, "")
        if not pattern.fullmatch(value):
            problems[field] = f"{value!r} is not valid: {message}"
    return problems


submission = {
    "username": "meera_n",
    "pin": "41100",
    "phone": "9876543210",
    "email": "meera@example",
    "date": "2026-13-01",
}

for field, message in validate(submission).items():
    print(f"{field:<10}{message}")

Performance on large text

import re
import time

text = ("word " * 200_000) + "target"

# Compile once, outside the loop
pattern = re.compile(r"target")

start = time.perf_counter()
pattern.search(text)
print(f"compiled search: {time.perf_counter() - start:.4f}s")

# A plain substring test is much faster when the pattern is fixed
start = time.perf_counter()
"target" in text
print(f"in operator:     {time.perf_counter() - start:.4f}s")

• Use in, startswith and split when the text is fixed.
• Compile patterns used repeatedly.
• Prefer [^x]* to .*? - it cannot backtrack across the boundary.
• Anchor patterns with ^ or \b so the engine can fail fast.
• Use finditer rather than findall on very large input.

Common mistakes

• Trying to parse HTML or JSON with a regex. Nested structures are beyond regular expressions - use a parser.
• Writing one pattern that does four jobs instead of four that each do one.
• Forgetting that sub returns a new string.
• Building a pattern from user input without re.escape.
• Validating with search instead of fullmatch.
• Editing files in bulk with no dry run and no backup.

Best practices

• Break text processing into named steps, each with one substitution.
• Compile patterns as module level constants in capitals.
• Use named groups and turn matches straight into dictionaries.
• Report the lines that did not match rather than silently dropping them.
• Give bulk editing scripts a dry run mode, on by default.
• Test against input that should fail, not only input that should pass.

Practice

1. Write a cleaner that strips markup, URLs and extra whitespace from a paragraph.
2. Parse a log format of your choice into records and report counts by level.
3. Write a redaction function masking emails, phone numbers and card numbers.
4. Convert names between snake case, camel case and kebab case in both directions.
5. Validate a five field form and report a specific message for each failing field.

Conclusion

Regular expressions are at their best on text that varies but follows a rule: log lines, identifiers, form fields, formatted records. Break the work into small named patterns, compile them, use named groups, and always report what failed to match.