[Python] Concurrent module

concurrent.futures

ThreadPoolExecutor

• Runs each of your workers in separate threads within the main process.
• Suitable for I/O-bound tasks like retrieving web resource.

import requests
from concurrent import futures

to_do = []

def get_resource(url):
  return requests.get(url)

""" First way """

with futures.ThreadPoolExecutor(max_workers = 3) as executor:
  for url in [URL1, URL2, URL3, ...]:
    # Non-blocking
    future = executor.submit(get_resource, url)
    to_do.append(future)

  for future in futures.as_completed(to_do):
    # It is Non-blocking since we use as_completed
    res = future.result()

""" Second way """

with futures.ThreadPoolExecutor(max_workers = 3) as executor:
  # Non-blocking
  results = executor.map(get_resource, [URL1, URL2, URL3, ...])

  # Block until future in results is completed
  # enumerate would call future.result()
  for i, res in enumerate(results):
    print(i, res)

ProcessPoolExecutor

• Runs each of your workers in its own separate child process.
• Suitable for CPU-bound tasks like calculation.

from concurrent import futures

to_do = []

def fib(n):
  if n < 2:
    return 1
  return fib(n - 1) + fib(n - 2)

""" First way """

with futures.ProcessPoolExecutor(max_workers = 3) as executor:
  for n in range(1, 10):
    # Non-blocking
    future = executor.submit(fib, n)
    to_do.append(future)

  for future in futures.as_completed(to_do):
    # It is Non-blocking since we use as_completed
    res = future.result()

""" Second way """

with futures.ProcessPoolExecutor(max_workers = 3) as executor:
  results = executor.map(fib, range(1, 10))

  # Block until future in results is completed
  # enumerate would call future.result()
  for i, res in enumerate(results):
    print(i, res)