What is Natural Language Processing?
Natural Language Processing (NLP) is a branch of AI that enables computers to understand, interpret, and generate human language. It powers chatbots, translation services, sentiment analysis, and search engines.
NLP Fundamentals
Text Preprocessing
Every NLP pipeline starts with cleaning and preparing text:
import re
from collections import Counter
def text_preprocessing(text):
"""Essential NLP text preprocessing steps"""
# Step 1: Lowercase
text = text.lower()
print(f"1. Lowercase: {text[:50]}...")
# Step 2: Remove special characters and numbers
text = re.sub(r'[^a-zA-Z\s]', '', text)
print(f"2. Cleaned: {text[:50]}...")
# Step 3: Tokenization (split into words)
tokens = text.split()
print(f"3. Tokens: {tokens[:8]}...")
# Step 4: Remove stop words
stop_words = {'the', 'a', 'an', 'is', 'are', 'was', 'in', 'on',
'at', 'to', 'for', 'of', 'and', 'or', 'but', 'with'}
filtered = [t for t in tokens if t not in stop_words]
print(f"4. No stop words: {filtered[:8]}...")
# Step 5: Simple stemming
def simple_stem(word):
for suffix in ['ing', 'ly', 'ed', 'ious', 'ies', 'ive', 'es', 's', 'ment']:
if word.endswith(suffix) and len(word) > len(suffix) + 2:
return word[:-len(suffix)]
return word
stemmed = [simple_stem(t) for t in filtered]
print(f"5. Stemmed: {stemmed[:8]}...")
return stemmed
# Example
text = "The quick brown foxes are jumping over the lazy dogs in the garden!"
result = text_preprocessing(text)Word Representations
Converting words to numbers that models can process:
import numpy as np
def word_representation_demo():
"""Different ways to represent words as numbers"""
# 1. One-Hot Encoding
vocab = ['king', 'queen', 'man', 'woman', 'cat', 'dog']
for word in vocab:
one_hot = [1 if v == word else 0 for v in vocab]
print(f"One-Hot('{word}'): {one_hot}")
print()
# 2. Bag of Words
documents = [
"the cat sat on the mat",
"the dog sat on the log",
]
all_words = set()
for doc in documents:
all_words.update(doc.split())
vocab = sorted(all_words)
print(f"Vocabulary: {vocab}")
for doc in documents:
bow = [doc.split().count(w) for w in vocab]
print(f"BoW('{doc[:20]}...'): {bow}")
print()
# 3. TF-IDF
def tfidf(documents):
"""Calculate TF-IDF scores"""
# Term Frequency
tf = []
for doc in documents:
words = doc.split()
total = len(words)
tf.append({w: words.count(w)/total for w in set(words)})
# Inverse Document Frequency
n_docs = len(documents)
idf = {}
all_words = set(w for doc in documents for w in doc.split())
for word in all_words:
containing = sum(1 for doc in documents if word in doc)
idf[word] = np.log(n_docs / containing)
return tf, idf
tf, idf = tfidf(documents)
print("TF-IDF scores:")
for word in sorted(idf.keys()):
avg_tf = sum(t.get(word, 0) for t in tf) / len(tf)
print(f" {word}: TF={avg_tf:.3f}, IDF={idf[word]:.3f}, TF-IDF={avg_tf*idf[word]:.3f}")
word_representation_demo()Modern NLP: Transformers
Sentiment Analysis with Transformers
from transformers import pipeline
# Pre-trained sentiment analysis
analyzer = pipeline("sentiment-analysis",
model="distilbert-base-uncased-finetuned-sst-2-english")
texts = [
"I absolutely love this product! It exceeded my expectations.",
"The service was terrible. I will never come back.",
"It's okay, nothing special but not bad either.",
"This is the best experience I've ever had!",
]
for text in texts:
result = analyzer(text)[0]
print(f"{result['label']:>8} ({result['score']:.3f}): {text[:50]}")Text Classification
from transformers import pipeline
# Zero-shot classification (no training data needed!)
classifier = pipeline("zero-shot-classification",
model="facebook/bart-large-mnli")
text = "The stock market crashed today as investors panicked"
candidate_labels = ["finance", "sports", "technology", "politics", "entertainment"]
result = classifier(text, candidate_labels)
print(f"Text: '{text}'")
for label, score in zip(result['labels'], result['scores']):
print(f" {label}: {score:.1%}")Named Entity Recognition (NER)
from transformers import pipeline
# Named Entity Recognition
ner = pipeline("ner", grouped_entities=True)
text = "Apple Inc. was founded by Steve Jobs in Cupertino, California in 1976."
entities = ner(text)
print(f"Text: '{text}'\nEntities found:")
for entity in entities:
print(f" {entity['word']:>20} → {entity['entity_group']:>5} ({entity['score']:.3f})")Building NLP Applications
Text Summarization
from transformers import pipeline
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
article = """
Artificial intelligence has made remarkable progress in recent years.
Large language models like GPT-4 and Claude can write code, compose essays,
and engage in complex reasoning. These models are trained on vast amounts
of text data and use the transformer architecture to process language.
The technology has applications in healthcare, education, finance, and
many other industries. However, concerns about bias, misinformation,
and job displacement remain important considerations.
"""
summary = summarizer(article, max_length=50, min_length=20, do_sample=False)
print("Summary:", summary[0]['summary_text'])Translation
from transformers import pipeline
translator = pipeline("translation_en_to_es",
model="Helsinki-NLP/opus-mt-en-es")
text = "Natural language processing is transforming how computers understand human language."
translation = translator(text)
print(f"English: {text}")
print(f"Spanish: {translation[0]['translation_text']}")Question Answering
from transformers import pipeline
qa = pipeline("question-answering",
model="distilbert-base-cased-distilled-squad")
context = """
The Transformer architecture was introduced in 2017 by Google researchers
in the paper 'Attention Is All You Need'. It revolutionized NLP by enabling
parallel processing of sequences and introducing the self-attention mechanism.
BERT, GPT, and other modern language models are all based on transformers.
"""
questions = [
"When was the Transformer introduced?",
"Who introduced the Transformer?",
"What paper introduced the Transformer?",
]
for question in questions:
result = qa(question=question, context=context)
print(f"Q: {question}")
print(f"A: {result['answer']} (confidence: {result['score']:.3f})\n")NLP Pipeline Architecture
class NLPPipeline:
"""Complete NLP pipeline for text processing"""
def __init__(self):
self.steps = [
("1. Raw Text", self.raw_text),
("2. Tokenization", self.tokenize),
("3. POS Tagging", self.pos_tag),
("4. Named Entities", self.extract_entities),
("5. Sentiment", self.analyze_sentiment),
]
def raw_text(self, text):
return text
def tokenize(self, text):
import re
tokens = re.findall(r'\b\w+\b', text.lower())
return tokens
def pos_tag(self, tokens):
# Simplified POS tagging
common_nouns = {'computer', 'language', 'model', 'data', 'system'}
common_verbs = {'is', 'are', 'was', 'has', 'can', 'learns', 'processes'}
tags = []
for token in tokens:
if token in common_nouns:
tags.append((token, 'NN'))
elif token in common_verbs:
tags.append((token, 'VB'))
else:
tags.append((token, 'NN')) # Default
return tags
def extract_entities(self, text):
# Simplified NER
entities = []
known_entities = {
'Google': 'ORG', 'Apple': 'ORG', 'Python': 'TECH',
'TensorFlow': 'TECH', 'BERT': 'MODEL', 'GPT': 'MODEL',
}
for entity, label in known_entities.items():
if entity.lower() in text.lower():
entities.append((entity, label))
return entities
def analyze_sentiment(self, text):
positive_words = {'good', 'great', 'excellent', 'amazing', 'love', 'best'}
negative_words = {'bad', 'terrible', 'awful', 'worst', 'hate', 'poor'}
words = set(text.lower().split())
pos_count = len(words & positive_words)
neg_count = len(words & negative_words)
if pos_count > neg_count:
return "POSITIVE"
elif neg_count > pos_count:
return "NEGATIVE"
return "NEUTRAL"
def process(self, text):
for step_name, step_func in self.steps:
result = step_func(text)
print(f"{step_name}: {result}")
return result
pipeline = NLPPipeline()
print("NLP Pipeline Demo:")
pipeline.process("Google BERT is an amazing language model. It processes data well.")Frequently Asked Questions
What is NLP used for in the real world?
NLP powers virtual assistants (Siri, Alexa), email spam filters, translation services (Google Translate), chatbots, sentiment analysis for brands, search engines, and medical text analysis.
What is the difference between NLP and machine learning?
NLP is a specific application area of AI focused on language. Machine learning is the broader technique used to build NLP systems. You use machine learning to build NLP models.
What is the best Python library for NLP?
HuggingFace Transformers is the most popular for modern NLP. NLTK and spaCy are excellent for traditional NLP tasks. Gensim is great for topic modeling and word embeddings.
How do transformers work in NLP?
Transformers use self-attention mechanisms to process entire sequences at once, capturing relationships between all words simultaneously. This enables better understanding of context compared to older sequential models like RNNs.
Want to build NLP projects? Join our Telegram Community for NLP tutorials, project ideas, and discussions about text processing and language AI.