1# Typical scenario in 2016:
2
3Dataset: 10 million sentence pairs (English-Hindi)
4Model: LSTM-based Seq2Seq  
5Hardware: 8 NVIDIA GPUs (expensive!)
6Training time: 2-3 WEEKS! 😱
7
8# If hyperparameter was wrong?
9# Wait another 2-3 weeks! 
10# Total experiments: 50+
11# Total time: 6 months+ just for one model!

1# ========================================
2# OLD WAY: LSTM-based Seq2Seq (2016)
3# ========================================
4# Library: TensorFlow 1.x or Keras
5
6import tensorflow as tf
7from tensorflow.keras.layers import LSTM, Dense, Embedding
8from tensorflow.keras.models import Model
9
10class OldSchoolTranslator:
11    """
12    2016-style translation model
13    Problems:
14    - Sequential processing (SLOW!)
15    - Vanishing gradients
16    - Limited context
17    - Hard to train
18    """
19    
20    def __init__(self, vocab_size=10000, embedding_dim=256, hidden_dim=512):
21        # Encoder: Reads input sequence one word at a time
22        self.encoder_embedding = Embedding(vocab_size, embedding_dim)
23        self.encoder_lstm = LSTM(
24            hidden_dim, 
25            return_state=True,  # Need final state for decoder
26            return_sequences=True
27        )
28        
29        # Decoder: Generates output one word at a time
30        self.decoder_embedding = Embedding(vocab_size, embedding_dim)
31        self.decoder_lstm = LSTM(hidden_dim, return_sequences=True)
32        self.output_layer = Dense(vocab_size, activation='softmax')
33    
34    def encode(self, input_sequence):
35        """
36        Process input ONE WORD AT A TIME
37        Word 1 → Word 2 → Word 3 → ... → Final State
38        
39        PROBLEM: By the time we reach word 100,
40                 we've "forgotten" word 1!
41        """
42        embedded = self.encoder_embedding(input_sequence)
43        
44        # LSTM processes sequentially - NO PARALLELIZATION!
45        # This is the BOTTLENECK
46        outputs, state_h, state_c = self.encoder_lstm(embedded)
47        
48        return outputs, [state_h, state_c]
49    
50    def decode(self, target_sequence, encoder_states):
51        """
52        Generate output ONE WORD AT A TIME
53        Can't predict word 5 until word 4 is generated!
54        """
55        embedded = self.decoder_embedding(target_sequence)
56        outputs = self.decoder_lstm(embedded, initial_state=encoder_states)
57        predictions = self.output_layer(outputs)
58        return predictions
59    
60    def train_step(self, source, target):
61        """
62        Training was PAINFUL:
63        - Gradient vanishing/exploding
64        - Teacher forcing required
65        - Weeks of training time
66        """
67        # Forward pass
68        encoder_outputs, encoder_states = self.encode(source)
69        predictions = self.decode(target, encoder_states)
70        
71        # Loss calculation
72        loss = tf.keras.losses.sparse_categorical_crossentropy(
73            target, predictions
74        )
75        
76        return loss
77
78# TRAINING TIME: 2-3 WEEKS on 8 GPUs! 😱
79# BLEU SCORE: ~25 (decent but not great)
80# CONTEXT: ~50-100 tokens before quality degrades

1# ========================================
2# NEW WAY: Modern Transformer (2024-2025)
3# ========================================
4# Libraries: PyTorch + HuggingFace Transformers
5
6import torch
7import torch.nn as nn
8import torch.nn.functional as F
9from transformers import AutoModelForCausalLM, AutoTokenizer
10
11# ==== USING PRE-TRAINED MODEL (RECOMMENDED) ====
12class ModernTranslator:
13    """
14    2024-style using pre-trained LLM
15    Just few lines of code!
16    """
17    
18    def __init__(self, model_name="google/gemma-2b"):
19        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
20        self.model = AutoModelForCausalLM.from_pretrained(
21            model_name,
22            torch_dtype=torch.bfloat16,  # Memory efficient
23            device_map="auto"  # Automatic GPU allocation
24        )
25    
26    def translate(self, text, source_lang="Hindi", target_lang="English"):
27        prompt = f"Translate from {source_lang} to {target_lang}: {text}\n\nTranslation:"
28        
29        inputs = self.tokenizer(prompt, return_tensors="pt").to("cuda")
30        
31        with torch.no_grad():
32            outputs = self.model.generate(
33                **inputs,
34                max_new_tokens=100,
35                temperature=0.7,
36                do_sample=True
37            )
38        
39        return self.tokenizer.decode(outputs[0], skip_special_tokens=True)
40
41# THAT'S IT! Pre-trained model does the heavy lifting!
42
43
44# ==== UNDERSTANDING THE INTERNALS ====
45class SimpleTransformerBlock(nn.Module):
46    """
47    Simplified Transformer block to understand the architecture
48    Real implementations are more complex but same principle!
49    """
50    
51    def __init__(self, d_model=512, n_heads=8, d_ff=2048, dropout=0.1):
52        super().__init__()
53        self.d_model = d_model
54        self.n_heads = n_heads
55        self.d_k = d_model // n_heads
56        
57        # Multi-Head Attention components
58        self.W_q = nn.Linear(d_model, d_model)  # Query projection
59        self.W_k = nn.Linear(d_model, d_model)  # Key projection
60        self.W_v = nn.Linear(d_model, d_model)  # Value projection
61        self.W_o = nn.Linear(d_model, d_model)  # Output projection
62        
63        # Feed-Forward Network
64        self.ffn = nn.Sequential(
65            nn.Linear(d_model, d_ff),
66            nn.GELU(),  # Modern activation (better than ReLU)
67            nn.Linear(d_ff, d_model)
68        )
69        
70        # Layer Normalization (Pre-LN is modern standard)
71        self.norm1 = nn.LayerNorm(d_model)
72        self.norm2 = nn.LayerNorm(d_model)
73        
74        self.dropout = nn.Dropout(dropout)
75    
76    def scaled_dot_product_attention(self, Q, K, V, mask=None):
77        """
78        THE CORE OF TRANSFORMER!
79        
80        Attention(Q, K, V) = softmax(QK^T / √d_k) × V
81        
82        - Q: What am I looking for? (Query)
83        - K: What do I have? (Key)
84        - V: What's my actual content? (Value)
85        """
86        # Step 1: Calculate attention scores
87        # Q @ K^T gives similarity between each query and all keys
88        scores = torch.matmul(Q, K.transpose(-2, -1))
89        
90        # Step 2: Scale by √d_k (prevents softmax saturation)
91        scores = scores / (self.d_k ** 0.5)
92        
93        # Step 3: Apply mask if needed (for decoder)
94        if mask is not None:
95            scores = scores.masked_fill(mask == 0, float('-inf'))
96        
97        # Step 4: Softmax to get attention weights (sum to 1)
98        attention_weights = F.softmax(scores, dim=-1)
99        
100        # Step 5: Weighted sum of values
101        output = torch.matmul(attention_weights, V)
102        
103        return output, attention_weights
104    
105    def multi_head_attention(self, x, mask=None):
106        """
107        Multiple attention "heads" looking at different aspects
108        
109        Head 1: Grammar relationships
110        Head 2: Semantic meaning
111        Head 3: Entity tracking
112        ... etc
113        """
114        batch_size, seq_len, _ = x.shape
115        
116        # Project to Q, K, V
117        Q = self.W_q(x)  # [batch, seq, d_model]
118        K = self.W_k(x)
119        V = self.W_v(x)
120        
121        # Split into multiple heads
122        # [batch, seq, d_model] → [batch, n_heads, seq, d_k]
123        Q = Q.view(batch_size, seq_len, self.n_heads, self.d_k).transpose(1, 2)
124        K = K.view(batch_size, seq_len, self.n_heads, self.d_k).transpose(1, 2)
125        V = V.view(batch_size, seq_len, self.n_heads, self.d_k).transpose(1, 2)
126        
127        # Apply attention to each head IN PARALLEL!
128        # This is why Transformer is fast - all heads computed together
129        attn_output, _ = self.scaled_dot_product_attention(Q, K, V, mask)
130        
131        # Concatenate heads back together
132        attn_output = attn_output.transpose(1, 2).contiguous()
133        attn_output = attn_output.view(batch_size, seq_len, self.d_model)
134        
135        # Final linear projection
136        output = self.W_o(attn_output)
137        
138        return output
139    
140    def forward(self, x, mask=None):
141        """
142        Full transformer block:
143        1. Multi-Head Attention + Residual + LayerNorm
144        2. Feed-Forward Network + Residual + LayerNorm
145        """
146        # Pre-LN Transformer (modern standard)
147        # Attention block with residual connection
148        normalized = self.norm1(x)
149        attention_output = self.multi_head_attention(normalized, mask)
150        x = x + self.dropout(attention_output)  # Residual connection
151        
152        # Feed-forward block with residual connection
153        normalized = self.norm2(x)
154        ffn_output = self.ffn(normalized)
155        x = x + self.dropout(ffn_output)  # Residual connection
156        
157        return x
158
159
160# ==== POSITIONAL ENCODING (RoPE - Modern Standard) ====
161class RotaryPositionalEmbedding(nn.Module):
162    """
163    RoPE - Rotary Position Embedding
164    Used by: LLaMA, Qwen, DeepSeek, Mistral
165    
166    Instead of adding position, ROTATE the embeddings!
167    Position difference = Angle difference
168    """
169    
170    def __init__(self, d_model, max_seq_len=8192, base=10000):
171        super().__init__()
172        
173        # Precompute rotation frequencies
174        inv_freq = 1.0 / (base ** (torch.arange(0, d_model, 2).float() / d_model))
175        self.register_buffer('inv_freq', inv_freq)
176        
177        # Precompute sin/cos for all positions
178        positions = torch.arange(max_seq_len).float()
179        freqs = torch.einsum('i,j->ij', positions, inv_freq)
180        
181        # [max_seq_len, d_model/2]
182        self.register_buffer('cos_cached', freqs.cos())
183        self.register_buffer('sin_cached', freqs.sin())
184    
185    def forward(self, x, seq_len):
186        """
187        Apply rotation to embeddings based on position
188        """
189        cos = self.cos_cached[:seq_len]
190        sin = self.sin_cached[:seq_len]
191        
192        # Split into pairs and rotate
193        x1, x2 = x[..., ::2], x[..., 1::2]
194        
195        # Apply rotation
196        rotated = torch.stack([
197            x1 * cos - x2 * sin,
198            x1 * sin + x2 * cos
199        ], dim=-1).flatten(-2)
200        
201        return rotated
202
203
204# ==== USING GOOGLE GEMINI API (Easiest!) ====
205from google import genai
206from google.genai import types
207
208def translate_with_gemini(text: str) -> str:
209    """
210    Modern way: Just use API!
211    No training, no infrastructure needed
212    """
213    client = genai.Client(api_key="YOUR_API_KEY")
214    
215    response = client.models.generate_content(
216        model="gemini-3-flash-preview",
217        contents=f"Translate to English: {text}",
218        config=types.GenerateContentConfig(
219            thinking_config=types.ThinkingConfig(thinking_level="minimal")
220        )
221    )
222    
223    return response.text
224
225# COMPARISON TABLE:
226# ┌────────────────┬────────────────┬────────────────┐
227# │    Metric      │   Old (2016)   │   New (2024)   │
228# ├────────────────┼────────────────┼────────────────┤
229# │ Training Time  │   2-3 weeks    │  Already done! │
230# │ Code Lines     │    500+        │     10-20      │
231# │ Context Length │   50-100       │  1M+ tokens    │
232# │ BLEU Score     │    ~25         │    45+         │
233# │ GPU Required   │   8 GPUs       │ API call only  │
234# └────────────────┴────────────────┴────────────────┘

1# ========================================
2# MODERN LIBRARIES FOR LLM DEVELOPMENT
3# ========================================
4
5# 1. HUGGING FACE TRANSFORMERS (Most Popular)
6# pip install transformers accelerate
7from transformers import pipeline
8
9# One line to load and use!
10translator = pipeline("translation_en_to_de", model="Helsinki-NLP/opus-mt-en-de")
11result = translator("Hello, how are you?")
12
13# 2. LANGCHAIN (For LLM Applications)
14# pip install langchain langchain-openai
15from langchain_openai import ChatOpenAI
16from langchain.prompts import ChatPromptTemplate
17
18llm = ChatOpenAI(model="gpt-4")
19prompt = ChatPromptTemplate.from_messages([
20    ("system", "You are a helpful translator."),
21    ("user", "Translate to Hindi: {text}")
22])
23chain = prompt | llm
24result = chain.invoke({"text": "Hello world"})
25
26# 3. GOOGLE GENAI SDK (For Gemini)
27# pip install google-genai
28from google import genai
29client = genai.Client(api_key="YOUR_KEY")
30response = client.models.generate_content(
31    model="gemini-3-flash-preview",
32    contents="Explain quantum computing"
33)
34
35# 4. ANTHROPIC SDK (For Claude)
36# pip install anthropic
37import anthropic
38client = anthropic.Anthropic(api_key="YOUR_KEY")
39response = client.messages.create(
40    model="claude-3-5-sonnet-20241022",
41    max_tokens=1024,
42    messages=[{"role": "user", "content": "Hello!"}]
43)
44
45# 5. OLLAMA (For Local LLMs)
46# Install ollama from ollama.ai
47# ollama pull llama3.2
48import ollama
49response = ollama.chat(
50    model='llama3.2',
51    messages=[{'role': 'user', 'content': 'Why is the sky blue?'}]
52)
53
54# 6. VLLM (For Fast Inference)
55# pip install vllm
56from vllm import LLM, SamplingParams
57llm = LLM(model="meta-llama/Llama-3.1-8B")
58outputs = llm.generate(["Hello, my name is"], SamplingParams(temperature=0.8))