Python Web Scraping Tutorial¶

View as ipynb file here.

In [ ]:
import pandas as pd
import requests
from bs4 import BeautifulSoup
In [ ]:
url = "https://www.maxpreps.com/nj/football/stat-leaders/offense/passing/yds/"
response = requests.get(url)
html_content = response.text
soup = BeautifulSoup(html_content, 'lxml') # or 'html.parser'
In [ ]:
headers = [a.get_text(strip=False) for a in soup.find_all('th')]
headers
Out[ ]:
['#', 'Name', 'Yds', 'Yds/G', 'Comp', 'Att', 'Pct', 'TD', 'Int', 'Rate', 'GP']
In [ ]:
dataframe = []
rows = [a for a in soup.find_all('tr')]

for row in rows:
    soup = BeautifulSoup(str(row), "html.parser")
    
    data = [td.get_text(strip=False) for td in soup.find_all(['td','span','a'], recursive=True)]
    if len(data)==1:continue
    del data[1] # removes the name, position, school merged cell
    dataframe += [data]
    print(data)

['1', 'Lukas Prock', 'QB', 'Hun (Princeton)', '2454', '409.0', '120', '163', '0.736', '27', '1', '153', '6']
['2', 'Marco Green', 'QB', 'PCHS (Paramus)', '1480', '296.0', '88', '153', '0.575', '10', '3', '104', '5']
['3', 'Jalen Parker', 'QB, SB', 'Winslow Township (Atco)', '1374', '229.0', '103', '168', '0.613', '11', '7', '92', '6']
['4', 'Matthew Rossi', 'QB, DE', 'Point Pleasant Beach', '1345', '224.2', '74', '124', '0.597', '13', '1', '129', '6']
['5', 'Danny Ferrauilo', 'QB, DB', 'Wayne Valley (Wayne)', '1334', '222.3', '96', '124', '0.774', '13', '3', '136', '6']
['6', 'Sam Meekings', 'QB', 'Voorhees (Glen Gardner)', '1271', '211.8', '76', '122', '0.623', '17', '8', '110', '6']
['7', 'Pierce Asfalg', 'QB, CB', 'Cedar Grove', '1261', '210.2', '98', '137', '0.715', '7', '2', '111', '6']
['8', 'Derek Zammit', 'QB', 'DePaul Catholic (Wayne)', '1242', '207.0', '85', '130', '0.654', '10', '4', '109', '6']
['9', 'Dylan Chiera', 'QB', 'Colonia', '1176', '196.0', '87', '147', '0.592', '12', '1', '109', '6']
['10', 'Ryan Corwin', 'QB', 'MRHS (Tinton Falls)', '1176', '196.0', '66', '112', '0.589', '10', '4', '110', '6']
['11', 'Jack Oconnell', 'QB', 'Glassboro', '1156', '192.7', '58', '100', '0.580', '19', '3', '126', '6']
['12', 'Jason Lajara', 'QB', 'Brick Memorial (Brick)', '1120', '280.0', '69', '95', '0.726', '12', '2', '143', '4']
['13', 'Carter Bujno', 'QB, WR', 'Hackettstown', '1108', '184.7', '49', '81', '0.605', '13', '5', '118', '6']
['14', 'Mikey Branagh', 'Mahwah', '1068', '178.0', '66', '102', '0.647', '11', '4', '119', '6']
['15', 'Zane Forest', 'QB', 'Oakcrest (Mays Landing)', '1031', '171.8', '58', '100', '0.580', '11', '4', '113', '6']
['16', 'David Oravetz', 'KHS (Woolwich Township)', '1002', '167.0', '61', '89', '0.685', '9', '3', '126', '6']
['17', 'Ryan Moye', 'QB', 'TPS (Martinsville)', '1001', '200.2', '61', '93', '0.656', '11', '3', '128', '5']
['18', 'Devon Khurana', 'QB', 'Hillsborough', '988', '164.7', '65', '120', '0.542', '5', '6', '75', '6']
['19', 'Trey Tagliaferri', 'QB', 'Bergen Catholic (Oradell)', '975', '195.0', '64', '102', '0.627', '10', '1', '123', '5']
['20', 'Casey Grusser', 'QB', 'Ramapo (Franklin Lakes)', '972', '194.4', '60', '80', '0.750', '8', '2', '138', '5']
['21', 'BlairCarter Gagneron', 'MRHS (Aberdeen)', '959', '159.8', '45', '80', '0.562', '6', '1', '119', '6']
['22', 'Cohl Stevens', 'QB', 'WHRHS (Washington)', '936', '187.2', '56', '93', '0.602', '9', '6', '100', '5']
['23', "A'rique Fleming", 'QB, P', 'Elizabeth', '910', '151.7', '86', '136', '0.632', '9', '6', '86', '6']
['24', 'Van Lefakis', 'QB, SS', 'Haddonfield', '902', '150.3', '52', '82', '0.634', '11', '6', '110', '6']
['25', 'Dane Collum', 'QB, DB', 'Pitman', '899', '179.8', '43', '60', '0.717', '10', '3', '133', '5']
['26', 'Dakota Lasater', 'QB, DB', 'FTHS (Freehold)', '885', '221.2', '61', '103', '0.592', '11', '3', '111', '4']
['27', 'Nico Sampson', 'QB', 'Bayonne', '881', '146.8', '67', '95', '0.705', '11', '1', '134', '6']
['28', 'Malakhai Mckenzie', 'QB', 'Paulsboro', '845', '140.8', '57', '85', '0.671', '9', '2', '125', '6']
['29', 'Cameron Hayes-Durina', 'QB, FS', 'Metuchen', '824', '137.3', '57', '88', '0.648', '5', '8', '76', '6']
['30', 'Luke Senatore', 'QB, ATH', 'Fort Lee', '817', '163.4', '42', '75', '0.560', '13', '4', '112', '5']
['31', 'Owen Robson', 'QB', 'MNHS (Middletown)', '812', '162.4', '67', '100', '0.670', '4', '7', '76', '5']
['32', 'Vinny Sacca', 'QB, SS', 'Delran', '802', '133.7', '53', '91', '0.582', '10', '4', '106', '6']
['33', "Ja'Quan Robinson", 'QB, LB', 'Rahway', '797', '132.8', '40', '76', '0.526', '7', '1', '115', '6']
['34', 'Gavin Gilardi', 'QB', 'Delbarton (Morristown)', '783', '130.5', '55', '78', '0.705', '10', '3', '126', '6']
['35', 'Josh Greenberg', 'QB, ATH', 'CHEHS (Cherry Hill)', '783', '130.5', '51', '74', '0.689', '11', '1', '138', '6']
['36', 'Jack Kovacs', 'QB, S', 'HPHS (East Hanover)', '773', '128.8', '36', '55', '0.655', '10', '4', '118', '6']
['37', 'Davion Ross-Ways', 'ATH', 'KIPPCN (Camden)', '749', '149.8', '36', '60', '0.600', '6', '3', '117', '5']
['38', 'Shane Reilley', 'QB, OLB', 'Holy Cross (Delran)', '746', '149.2', '33', '55', '0.600', '10', '3', '121', '5']
['39', 'Jake Lewin', 'QB, FS', 'Cresskill', '737', '147.4', '56', '99', '0.566', '10', '1', '110', '5']
['40', 'Connor Acker', 'QB, DB', 'Howell (Farmingdale)', '715', '143.0', '56', '105', '0.533', '7', '4', '81', '5']
['41', 'Niko Jimenez', 'CB, QB', 'Woodbury', '713', '118.8', '59', '124', '0.476', '6', '8', '55', '6']
['42', 'Dylan Boehm', 'QB', 'Somerville', '699', '116.5', '49', '84', '0.583', '6', '5', '84', '6']
['43', 'JAMES WOLF', 'QB', 'Wayne Hills (Wayne)', '692', '115.3', '51', '68', '0.750', '8', '1', '140', '6']
['44', 'John Antrilli', 'QB', 'Collingswood', '692', '115.3', '50', '85', '0.588', '4', '4', '81', '6']
['45', 'David Smith', 'QB, CB', 'Butler', '691', '138.2', '49', '74', '0.662', '10', '4', '113', '5']
['46', 'Cole Aquino', 'QB', 'WTHS (Sewell)', '682', '113.7', '48', '69', '0.696', '11', '2', '129', '6']
['47', 'Quinton Defaria', 'QB, SS', 'Princeton', '681', '113.5', '52', '100', '0.520', '6', '4', '77', '6']
['48', 'Nazyr Negron', 'QB, S', 'Pennsauken', '673', '112.2', '42', '68', '0.618', '6', '5', '94', '6']
['49', 'Gary Battaglini, Jr', 'QB', 'Sterling (Somerdale)', '647', '129.4', '44', '62', '0.710', '11', '3', '124', '5']
['50', 'Gavin Yablonowitz', 'QB', 'Robbinsville', '646', '323.0', '49', '70', '0.700', '7', '1', '126', '2']
In [ ]:
# headers don't have position and school as they are in the name column so for this example I had to define it manually
columns = ['Rank', 'Name', 'Position', 'School', 'Yards', 'Avg', 'Comp', 'Att', 'Pct', 'TD', 'Int', 'Long', 'Games']
df = pd.DataFrame(dataframe, columns = columns)
In [69]:
df
Out[69]:
Rank Name Position School Yards Avg Comp Att Pct TD Int Long Games
0 1 Lukas Prock QB Hun (Princeton) 2454 409.0 120 163 0.736 27 1 153 6
1 2 Marco Green QB PCHS (Paramus) 1480 296.0 88 153 0.575 10 3 104 5
2 3 Jalen Parker QB, SB Winslow Township (Atco) 1374 229.0 103 168 0.613 11 7 92 6
3 4 Matthew Rossi QB, DE Point Pleasant Beach 1345 224.2 74 124 0.597 13 1 129 6
4 5 Danny Ferrauilo QB, DB Wayne Valley (Wayne) 1334 222.3 96 124 0.774 13 3 136 6
5 6 Sam Meekings QB Voorhees (Glen Gardner) 1271 211.8 76 122 0.623 17 8 110 6
6 7 Pierce Asfalg QB, CB Cedar Grove 1261 210.2 98 137 0.715 7 2 111 6
7 8 Derek Zammit QB DePaul Catholic (Wayne) 1242 207.0 85 130 0.654 10 4 109 6
8 9 Dylan Chiera QB Colonia 1176 196.0 87 147 0.592 12 1 109 6
9 10 Ryan Corwin QB MRHS (Tinton Falls) 1176 196.0 66 112 0.589 10 4 110 6
10 11 Jack Oconnell QB Glassboro 1156 192.7 58 100 0.580 19 3 126 6
11 12 Jason Lajara QB Brick Memorial (Brick) 1120 280.0 69 95 0.726 12 2 143 4
12 13 Carter Bujno QB, WR Hackettstown 1108 184.7 49 81 0.605 13 5 118 6
13 14 Mikey Branagh Mahwah 1068 178.0 66 102 0.647 11 4 119 6 None
14 15 Zane Forest QB Oakcrest (Mays Landing) 1031 171.8 58 100 0.580 11 4 113 6
15 16 David Oravetz KHS (Woolwich Township) 1002 167.0 61 89 0.685 9 3 126 6 None
16 17 Ryan Moye QB TPS (Martinsville) 1001 200.2 61 93 0.656 11 3 128 5
17 18 Devon Khurana QB Hillsborough 988 164.7 65 120 0.542 5 6 75 6
18 19 Trey Tagliaferri QB Bergen Catholic (Oradell) 975 195.0 64 102 0.627 10 1 123 5
19 20 Casey Grusser QB Ramapo (Franklin Lakes) 972 194.4 60 80 0.750 8 2 138 5
20 21 BlairCarter Gagneron MRHS (Aberdeen) 959 159.8 45 80 0.562 6 1 119 6 None
21 22 Cohl Stevens QB WHRHS (Washington) 936 187.2 56 93 0.602 9 6 100 5
22 23 A'rique Fleming QB, P Elizabeth 910 151.7 86 136 0.632 9 6 86 6
23 24 Van Lefakis QB, SS Haddonfield 902 150.3 52 82 0.634 11 6 110 6
24 25 Dane Collum QB, DB Pitman 899 179.8 43 60 0.717 10 3 133 5
25 26 Dakota Lasater QB, DB FTHS (Freehold) 885 221.2 61 103 0.592 11 3 111 4
26 27 Nico Sampson QB Bayonne 881 146.8 67 95 0.705 11 1 134 6
27 28 Malakhai Mckenzie QB Paulsboro 845 140.8 57 85 0.671 9 2 125 6
28 29 Cameron Hayes-Durina QB, FS Metuchen 824 137.3 57 88 0.648 5 8 76 6
29 30 Luke Senatore QB, ATH Fort Lee 817 163.4 42 75 0.560 13 4 112 5
30 31 Owen Robson QB MNHS (Middletown) 812 162.4 67 100 0.670 4 7 76 5
31 32 Vinny Sacca QB, SS Delran 802 133.7 53 91 0.582 10 4 106 6
32 33 Ja'Quan Robinson QB, LB Rahway 797 132.8 40 76 0.526 7 1 115 6
33 34 Gavin Gilardi QB Delbarton (Morristown) 783 130.5 55 78 0.705 10 3 126 6
34 35 Josh Greenberg QB, ATH CHEHS (Cherry Hill) 783 130.5 51 74 0.689 11 1 138 6
35 36 Jack Kovacs QB, S HPHS (East Hanover) 773 128.8 36 55 0.655 10 4 118 6
36 37 Davion Ross-Ways ATH KIPPCN (Camden) 749 149.8 36 60 0.600 6 3 117 5
37 38 Shane Reilley QB, OLB Holy Cross (Delran) 746 149.2 33 55 0.600 10 3 121 5
38 39 Jake Lewin QB, FS Cresskill 737 147.4 56 99 0.566 10 1 110 5
39 40 Connor Acker QB, DB Howell (Farmingdale) 715 143.0 56 105 0.533 7 4 81 5
40 41 Niko Jimenez CB, QB Woodbury 713 118.8 59 124 0.476 6 8 55 6
41 42 Dylan Boehm QB Somerville 699 116.5 49 84 0.583 6 5 84 6
42 43 JAMES WOLF QB Wayne Hills (Wayne) 692 115.3 51 68 0.750 8 1 140 6
43 44 John Antrilli QB Collingswood 692 115.3 50 85 0.588 4 4 81 6
44 45 David Smith QB, CB Butler 691 138.2 49 74 0.662 10 4 113 5
45 46 Cole Aquino QB WTHS (Sewell) 682 113.7 48 69 0.696 11 2 129 6
46 47 Quinton Defaria QB, SS Princeton 681 113.5 52 100 0.520 6 4 77 6
47 48 Nazyr Negron QB, S Pennsauken 673 112.2 42 68 0.618 6 5 94 6
48 49 Gary Battaglini, Jr QB Sterling (Somerdale) 647 129.4 44 62 0.710 11 3 124 5
49 50 Gavin Yablonowitz QB Robbinsville 646 323.0 49 70 0.700 7 1 126 2