1"""
2Python Generators and Iterators
3
4Demonstrates:
5- Generator functions with yield
6- Generator expressions
7- send() and bidirectional generators
8- yield from delegation
9- itertools module
10- Infinite generators
11- Generator pipelines
12"""
13
14import itertools
15from typing import Iterator, Any
16
17
18def section(title: str) -> None:
19 """Print a section header."""
20 print("\n" + "=" * 60)
21 print(f" {title}")
22 print("=" * 60)
23
24
25# =============================================================================
26# Basic Generator Functions
27# =============================================================================
28
29section("Basic Generator Functions")
30
31
32def simple_generator():
33 """Simple generator yielding values."""
34 print(" Yielding 1")
35 yield 1
36 print(" Yielding 2")
37 yield 2
38 print(" Yielding 3")
39 yield 3
40 print(" Generator exhausted")
41
42
43gen = simple_generator()
44print(f"Generator object: {gen}")
45print(f"Type: {type(gen)}")
46
47for value in gen:
48 print(f" Received: {value}")
49
50
51def countdown(n: int) -> Iterator[int]:
52 """Countdown from n to 1."""
53 while n > 0:
54 yield n
55 n -= 1
56
57
58print("\nCountdown from 5:")
59for i in countdown(5):
60 print(f" {i}")
61
62
63# =============================================================================
64# Generator Expressions
65# =============================================================================
66
67section("Generator Expressions")
68
69# List comprehension - creates entire list in memory
70squares_list = [x**2 for x in range(10)]
71print(f"List comprehension: {squares_list}")
72
73# Generator expression - lazy evaluation
74squares_gen = (x**2 for x in range(10))
75print(f"Generator expression: {squares_gen}")
76print(f"First 5 squares: {list(itertools.islice(squares_gen, 5))}")
77
78
79# Memory efficiency example
80def memory_comparison():
81 """Compare memory usage of list vs generator."""
82 import sys
83
84 n = 1000
85 list_comp = [x**2 for x in range(n)]
86 gen_exp = (x**2 for x in range(n))
87
88 list_size = sys.getsizeof(list_comp)
89 gen_size = sys.getsizeof(gen_exp)
90
91 print(f" List ({n} items): {list_size} bytes")
92 print(f" Generator: {gen_size} bytes")
93 print(f" Memory saved: {list_size - gen_size} bytes")
94
95
96memory_comparison()
97
98
99# =============================================================================
100# Fibonacci Generator
101# =============================================================================
102
103section("Fibonacci Generator")
104
105
106def fibonacci() -> Iterator[int]:
107 """Infinite Fibonacci sequence."""
108 a, b = 0, 1
109 while True:
110 yield a
111 a, b = b, a + b
112
113
114print("First 10 Fibonacci numbers:")
115for i, fib in enumerate(fibonacci()):
116 if i >= 10:
117 break
118 print(f" F({i}) = {fib}")
119
120
121# =============================================================================
122# Generator with send()
123# =============================================================================
124
125section("Bidirectional Generators with send()")
126
127
128def accumulator() -> Iterator[int]:
129 """Generator that accumulates sent values."""
130 total = 0
131 while True:
132 value = yield total
133 if value is not None:
134 total += value
135 print(f" Received {value}, total now {total}")
136
137
138acc = accumulator()
139next(acc) # Prime the generator
140print(f" send(10) -> {acc.send(10)}")
141print(f" send(20) -> {acc.send(20)}")
142print(f" send(5) -> {acc.send(5)}")
143
144
145def running_average() -> Iterator[float]:
146 """Calculate running average of sent values."""
147 total = 0.0
148 count = 0
149 while True:
150 value = yield total / count if count > 0 else 0.0
151 if value is not None:
152 total += value
153 count += 1
154
155
156avg = running_average()
157next(avg) # Prime
158print(f"\n Running average:")
159for val in [10, 20, 30, 40]:
160 result = avg.send(val)
161 print(f" After {val}: {result:.2f}")
162
163
164# =============================================================================
165# yield from - Generator Delegation
166# =============================================================================
167
168section("yield from - Generator Delegation")
169
170
171def generator_a():
172 """First generator."""
173 yield 1
174 yield 2
175
176
177def generator_b():
178 """Second generator."""
179 yield 3
180 yield 4
181
182
183def combined_manual():
184 """Manual delegation."""
185 for value in generator_a():
186 yield value
187 for value in generator_b():
188 yield value
189
190
191def combined_yield_from():
192 """Delegation with yield from."""
193 yield from generator_a()
194 yield from generator_b()
195
196
197print("Manual delegation:")
198print(f" {list(combined_manual())}")
199
200print("yield from delegation:")
201print(f" {list(combined_yield_from())}")
202
203
204# =============================================================================
205# itertools - Combinatoric Generators
206# =============================================================================
207
208section("itertools - Combinatoric Generators")
209
210# chain - concatenate iterables
211print("itertools.chain([1,2], [3,4], [5,6]):")
212print(f" {list(itertools.chain([1, 2], [3, 4], [5, 6]))}")
213
214# islice - slice an iterator
215print("\nitertools.islice(range(10), 2, 8, 2):")
216print(f" {list(itertools.islice(range(10), 2, 8, 2))}")
217
218# groupby - group consecutive items
219data = [1, 1, 2, 2, 2, 3, 1, 1]
220print(f"\nitertools.groupby({data}):")
221for key, group in itertools.groupby(data):
222 print(f" {key}: {list(group)}")
223
224# product - cartesian product
225print("\nitertools.product('AB', '12'):")
226for item in itertools.product('AB', '12'):
227 print(f" {item}")
228
229# combinations
230print("\nitertools.combinations('ABCD', 2):")
231for combo in itertools.combinations('ABCD', 2):
232 print(f" {combo}")
233
234
235# =============================================================================
236# Infinite Generators
237# =============================================================================
238
239section("Infinite Generators")
240
241
242def cycle_colors() -> Iterator[str]:
243 """Infinite cycle of colors."""
244 colors = ['red', 'green', 'blue']
245 while True:
246 yield from colors
247
248
249print("First 7 colors from infinite cycle:")
250for i, color in enumerate(cycle_colors()):
251 if i >= 7:
252 break
253 print(f" {i}: {color}")
254
255
256# Using itertools.cycle
257print("\nitertools.cycle (first 5):")
258for i, color in enumerate(itertools.cycle(['red', 'green', 'blue'])):
259 if i >= 5:
260 break
261 print(f" {color}")
262
263# itertools.count
264print("\nitertools.count(start=10, step=5) (first 6):")
265for i, num in enumerate(itertools.count(start=10, step=5)):
266 if i >= 6:
267 break
268 print(f" {num}")
269
270
271# =============================================================================
272# Generator Pipelines
273# =============================================================================
274
275section("Generator Pipelines")
276
277
278def read_numbers():
279 """Simulate reading numbers from source."""
280 yield from range(1, 21)
281
282
283def filter_even(numbers: Iterator[int]) -> Iterator[int]:
284 """Filter even numbers."""
285 for n in numbers:
286 if n % 2 == 0:
287 yield n
288
289
290def square(numbers: Iterator[int]) -> Iterator[int]:
291 """Square each number."""
292 for n in numbers:
293 yield n ** 2
294
295
296def take(n: int, iterable: Iterator[Any]) -> Iterator[Any]:
297 """Take first n items."""
298 for i, item in enumerate(iterable):
299 if i >= n:
300 break
301 yield item
302
303
304# Build pipeline
305pipeline = take(5, square(filter_even(read_numbers())))
306result = list(pipeline)
307print(f"Pipeline: read -> filter_even -> square -> take(5)")
308print(f"Result: {result}")
309
310
311# =============================================================================
312# File Processing with Generators
313# =============================================================================
314
315section("File Processing with Generators")
316
317
318def generate_log_lines():
319 """Simulate log file lines."""
320 logs = [
321 "INFO: Application started",
322 "DEBUG: Loading configuration",
323 "ERROR: Connection failed",
324 "INFO: Retrying connection",
325 "ERROR: Timeout exceeded",
326 "INFO: Application stopped"
327 ]
328 yield from logs
329
330
331def filter_errors(lines: Iterator[str]) -> Iterator[str]:
332 """Filter ERROR lines."""
333 for line in lines:
334 if 'ERROR' in line:
335 yield line
336
337
338def extract_message(lines: Iterator[str]) -> Iterator[str]:
339 """Extract message after colon."""
340 for line in lines:
341 if ':' in line:
342 yield line.split(':', 1)[1].strip()
343
344
345print("Error messages from logs:")
346error_pipeline = extract_message(filter_errors(generate_log_lines()))
347for msg in error_pipeline:
348 print(f" - {msg}")
349
350
351# =============================================================================
352# Generator State Preservation
353# =============================================================================
354
355section("Generator State Preservation")
356
357
358def stateful_generator():
359 """Generator maintains state between yields."""
360 state = {"count": 0}
361
362 while True:
363 state["count"] += 1
364 received = yield f"Call #{state['count']}"
365 if received:
366 print(f" Received: {received}")
367
368
369gen = stateful_generator()
370print(next(gen))
371print(gen.send("hello"))
372print(next(gen))
373print(gen.send("world"))
374
375
376# =============================================================================
377# Generator Best Practices
378# =============================================================================
379
380section("Generator Best Practices")
381
382
383def process_large_dataset(limit: int = 10) -> Iterator[dict]:
384 """
385 Process large dataset lazily.
386
387 Best practices:
388 - Yield one item at a time
389 - Don't accumulate all results
390 - Let caller control iteration
391 """
392 for i in range(limit):
393 # Simulate expensive processing
394 record = {
395 'id': i,
396 'value': i ** 2,
397 'processed': True
398 }
399 yield record
400
401
402print("Processing records lazily:")
403for record in process_large_dataset(5):
404 print(f" {record}")
405
406
407# =============================================================================
408# Summary
409# =============================================================================
410
411section("Summary")
412
413print("""
414Generator patterns covered:
4151. Basic generators - yield keyword
4162. Generator expressions - memory-efficient iteration
4173. send() - bidirectional communication
4184. yield from - delegation to sub-generators
4195. itertools - powerful combinatoric generators
420 - chain, islice, groupby, product, combinations
4216. Infinite generators - cycle, count
4227. Generator pipelines - composable data processing
4238. State preservation - generators maintain local state
424
425Generators provide:
426- Memory efficiency (lazy evaluation)
427- Infinite sequences
428- Pipeline processing
429- Stateful iteration
430- Clean, readable code
431""")