diff options
author | brian.quinlan <devnull@localhost> | 2009-05-04 21:02:48 +0000 |
---|---|---|
committer | brian.quinlan <devnull@localhost> | 2009-05-04 21:02:48 +0000 |
commit | a12b38ce97773a42379c032cb4003411c094c030 (patch) | |
tree | 48b6a806f5e7320eec8cd4fe15b5a55cb570ff67 /crawl.py | |
parent | 3b24ddaa96d138bac457f5d394c3a71947ad2d15 (diff) | |
download | futures-a12b38ce97773a42379c032cb4003411c094c030.tar.gz |
First maybe-working version of process pools.
Diffstat (limited to 'crawl.py')
-rw-r--r-- | crawl.py | 8 |
1 files changed, 4 insertions, 4 deletions
@@ -11,7 +11,7 @@ URLS = ['http://www.google.com/', 'http://www.thisurlprobablydoesnotexist.com', 'http://www.slashdot.org/', 'http://www.python.org/', - 'http://www.sweetapp.com/'] + 'http://www.sweetapp.com/'] * 1000 def load_url(url, timeout): return urllib.request.urlopen(url, timeout=timeout).read() @@ -25,14 +25,14 @@ def download_urls(urls, timeout=60): pass return url_to_content -executor = futures.thread.ThreadPoolExecutor(max_threads=100) +executor = futures.ProcessPoolExecutor(100) def download_urls_with_futures(urls, timeout=60): url_to_content = {} fs = executor.run( (functools.partial(load_url, url, timeout) for url in urls), timeout=timeout) - for url, future in zip(urls, fs.result_futures()): + for url, future in zip(urls, fs.successful_futures()): url_to_content[url] = future.result() return url_to_content -print(download_urls(URLS)) +print(download_urls_with_futures(URLS)) |