I've updated Dotancohen's answer to be asynchronous, report progress, and save everything to a csv.
A reminder that if you crank up the max connections, you risk getting throttled.
#!/usr/bin/python3
"""
(C) 2023 Christopher Overbeck
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License version 3.
http://www.gnu.org/licenses/gpl-3.0.html
"""
import re
import time
from bs4 import BeautifulSoup
import aiohttp
import asyncio
import csv
# extension of Semaphore that also throttles the number of requests per second
class ThrottledSemaphore(asyncio.Semaphore):
def __init__(self, value, rate):
super().__init__(value)
self.rate = rate
self.last_request = 0
async def acquire(self):
await super().acquire()
now = time.time()
if now - self.last_request < self.rate:
await asyncio.sleep(self.rate - (now - self.last_request))
self.last_request = time.time()
async def main():
# prints the mass launched for each year. Also reports number of launches with unknown (0) mass
start_url = 'http://space.skyrocket.de/directories/chronology.htm'
max_concurrent_connections = 5
max_connections_per_second = 10
semaphore = ThrottledSemaphore(max_concurrent_connections, 1/max_connections_per_second)
async with aiohttp.ClientSession() as session:
year_links = await get_year_links(session, start_url)
coro = [get_year(semaphore, session, link) for link in year_links]
tasks = [asyncio.create_task(c) for c in coro]
done, pending = await asyncio.wait(tasks)
years = [task.result() for task in done]
print("---")
for year in years:
print(f"{year[0]}: {year[1]} KG, {year[2]} launches, {year[3]} unknown mass")
print("---")
with open('launches.csv', 'w') as f:
writer = csv.writer(f)
writer.writerow(['year', 'total_mass', 'num_launches', 'num_unknown_mass'])
for year in years:
writer.writerow([year[0], year[1], year[2], year[3]])
while True:
await asyncio.sleep(1)
async def get_year_links(client_session: aiohttp.ClientSession, start_url: str):
"""Returns a list of links to each year's launches"""
year_links = []
async with client_session.get(start_url) as response:
html = await response.text()
parsed_page = BeautifulSoup(html, features="html.parser")
links = parsed_page.find_all('a')
for link in links:
link = str(link)
if 'doc_chr' in link:
tmp_link = link[link.find('"')+1:]
tmp_link = tmp_link[:tmp_link.find('"')]
year_links.append(tmp_link)
return year_links
class Counter:
def __init__(self, max_count: int, label: str = None):
self.count = 0
self.max_count = max_count
self.label = label
def increment(self):
self.count += 1
print(f"{self.label} {self.count}/{self.max_count}")
async def get_year(semaphore: asyncio.Semaphore, client_session: aiohttp.ClientSession, link: str):
"""Returns (year, total_mass, num_launches, num_unknown_mass) for the passed year link"""
# links look like ../doc_chr/lau1957.htm
year = int(re.search(r'(\d+)', link).group(0))
year_launches = []
async with semaphore:
async with client_session.get('http://space.skyrocket.de/' + link[3:]) as response:
html = await response.text()
parsed_page = BeautifulSoup(html, features="html.parser")
links = parsed_page.find_all('a')
for link in links:
link = str(link)
if 'doc_sdat' in link:
tmp_link = link[link.find('"')+1:]
tmp_link = tmp_link[:tmp_link.find('"')]
year_launches.append(tmp_link)
counter = Counter(len(year_launches), f"{year}")
tasks = [asyncio.create_task(get_launch_mass(semaphore, client_session, launch, counter)) for launch in year_launches]
done, pending = await asyncio.wait(tasks)
total_mass = [task.result() for task in done]
return (year, sum(total_mass), len(total_mass), total_mass.count(0))
async def get_launch_mass(semaphore: asyncio.Semaphore, client_session: aiohttp.ClientSession, launch_url: str, counter: Counter):
"""Returns the mass of a given launch
0 if the mass is unknown"""
mass = 0
try:
async with semaphore:
# print(f"Getting mass for {launch_url}")
async with client_session.get('http://space.skyrocket.de/' + launch_url[3:]) as response:
html = await response.text()
parsed_page = BeautifulSoup(html, features="html.parser")
for pre_mass_element in parsed_page.find_all(text='Mass:'):
mass_element = pre_mass_element.find_next()
mass = int(re.search(r'(\d+)', mass_element.text).group(0))
# print(f"Got mass of {mass}KG for {launch_url}")
except Exception:
# print(f"Error getting mass for {launch_url}")
pass
counter.increment()
return mass
if __name__ == '__main__':
asyncio.run(main())
Further work would be to check what data is already in the .csv and not re-download it.
Latest numbers as of 33/7/2023 are:
year kilograms launches launches (unknown mass) launches (known Mass) tons
1957 593 3 0 3 0.593
1958 7949 28 0 28 7.949
1959 1807 26 11 15 1.807
1960 6247 42 22 20 6.247
1961 24730 57 29 28 24.73
1962 41195 97 38 59 41.195
1963 39151 92 44 48 39.151
1964 136533 123 51 72 136.533
1965 168776 175 63 112 168.776
1966 149511 169 78 91 149.511
1967 107891 172 76 96 107.891
1968 206164 161 72 89 206.164
1969 201531 150 83 67 201.531
1970 203083 142 44 98 203.083
1971 363504 169 35 134 363.504
1972 345503 139 25 114 345.503
1973 467589 146 26 120 467.589
1974 262949 139 33 106 262.949
1975 324619 163 30 133 324.619
1976 310576 161 31 130 310.576
1977 294836 142 37 105 294.836
1978 314011 168 25 143 314.011
1979 309134 130 20 110 309.134
1980 290488 147 34 113 290.488
1981 317619 163 30 133 317.619
1982 306014 162 38 124 306.014
1983 298387 162 44 118 298.387
1984 235147 174 69 105 235.147
1985 201050 170 73 97 201.05
1986 173208 151 54 97 173.208
1987 292243 139 56 83 292.243
1988 269515 143 57 86 269.515
1989 212900 140 50 90 212.9
1990 215418 171 52 119 215.418
1991 201880 147 41 106 201.88
1992 211497 139 36 103 211.497
1993 152476 117 36 81 152.476
1994 211163 128 36 92 211.163
1995 198780 108 38 70 198.78
1996 171492 107 32 75 171.492
1997 241191 167 23 144 241.191
1998 205990 189 26 163 205.99
1999 183739 144 14 130 183.739
2000 243520 130 33 97 243.52
2001 161116 92 22 70 161.116
2002 200274 99 18 81 200.274
2003 144080 92 10 82 144.08
2004 146822 77 11 66 146.822
2005 199409 78 12 66 199.409
2006 185818 117 23 94 185.818
2007 195537 121 16 105 195.537
2008 230607 109 18 91 230.607
2009 213258 134 34 100 213.258
2010 203525 130 33 97 203.525
2011 283264 141 29 112 283.264
2012 284184 140 31 109 284.184
2013 298580 216 36 180 298.58
2014 258115 302 134 168 258.115
2015 289758 271 108 163 289.758
2016 289479 227 76 151 289.479
2017 322093 489 111 378 322.093
2018 298791 486 147 339 298.791
2019 318977 519 157 362 318.977
2020 475129 1316 166 1150 475.129
2021 610041 1891 370 1521 610.041
2022 917148 2566 361 2205 917.148
2023 368809 873 160 713 368.809
Summing to 16016.413 metric tons total.