python - Webscraping with a loop returns only a single element

Question

When I run a for loop to collect elements within a <div> tag it only returns the first from a list of all with the same class.

For example:

r = requests.get("https://one-versus-one.com/en/rankings/all/statistics")

soup = BeautifulSoup(r.content, 'lxml')

data = {
    'players': [],
    'club': [],
    'rank': []
}
def getstuff(soup):
    products = soup.find_all('div', {'class':'rankings-table'})
    for name in products:
        players = name.find('div', {'class':'player-name rankings-table__player-name'}).text
        club = name.find('span', {'class':'rankings-table__club-name'}).text
        rank = name.find('div', {'class':'rankings-table-cell value rankings-table__value'}).text.strip()
        data['players'] = players
        data['club'] = club
        data['rank'] = rank
        print(data)

getstuff(soup)

This returns:

{'players': 'Lionel Messi', 'club': 'Barcelona', 'rank': '100'}

Where I expected all players, clubs and ranks to be printed within the page.

Answer 1

You can try this:

import requests
from bs4 import BeautifulSoup

r = requests.get("https://one-versus-one.com/en/rankings/all/statistics")
soup = BeautifulSoup(r.content, 'lxml')

data = {'players': [],'club': [],'rank': []}

def getstuff(soup):
    products = soup.find('div', {'class':'rankings-table'}).find_all("a")
    for name in products:
        players = name.find('div', {'class':'player-name rankings-table__player-name'}).text
        club = name.find('span', {'class':'rankings-table__club-name'}).text
        rank = name.find('div', {'class':'rankings-table-cell value rankings-table__value'}).text.strip()
        data['players'].append(players)
        data['club'].append(club)
        data['rank'].append(rank)
    print(data)

getstuff(soup)
"""
{'players': ['Lionel Messi', 'Junior Neymar', 'Robert Lewandowski', 'Joao Cancelo', 'Kevin de Bruyne', 'Rodri', 'Jesse Lingard', 'Riyad Mahrez', 'Ilkay Gundogan', 'John Stones'], 'club': ['Barcelona', 'Paris Saint-Germain', 'Bayern Munich', 'Manchester City', 'Manchester City', 'Manchester City', 'West Ham United', 'Manchester City', 'Manchester City', 'Manchester City'], 'rank': ['100', '95', '93', '92', '91', '90', '90', '89', '88', '88']}
"""

You have to use .find_all("a") to get info about all players. And additional you're just making adding new player in data['players'] insted of adding new player and for club, rank same.