Using concurrent.futures to do Multithreading

Using concurrent.futures to download a list of urls and store the result in another list.

import concurrent.futures

import requests

def get_requests(url, data_list):

    resp = requests.get(url)

    data_list.append(resp.text)

URLs = [https://cnn.com”, https://aylanetworks.com”]

NUMBER_OF_THREADS = 10

data_list = []

with concurrent.futures.ThreadPoolExecutor(max_workers=NUMBER_OF_THREADS) as executor:

    future_to_url = {executor.submit(get_requests, url, data_list):

                     url for url in URLs}

print data_list

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google+ photo

You are commenting using your Google+ account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

w

Connecting to %s