python - Selenium Scraping Javascript Table

Question

I am stuggling to scrape as per code below. Would apprciate it if someone can have a look at what I am missing? Regards PyProg70

from selenium import webdriver
from selenium.webdriver import FirefoxOptions
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
from bs4 import BeautifulSoup
import pandas as pd
import re, time

binary = FirefoxBinary('/usr/bin/firefox')
opts = FirefoxOptions()
opts.add_argument("--headless")

browser = webdriver.Firefox(options=opts, firefox_binary=binary)
browser.implicitly_wait(10)

url = 'http://tenderbulletin.eskom.co.za/'
browser.get(url)

html = browser.page_source
soup = BeautifulSoup(html, 'lxml')

print(soup.prettify())

Answer 1

not Java but Javascript. it dynamic page you need to wait and check if Ajax finished the request and content rendered using WebDriverWait.

....
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait 
from selenium.webdriver.support import expected_conditions as EC

.....
browser.get(url)

# wait max 30 second until table loaded
WebDriverWait(browser, 30).until(EC.presence_of_element_located((By.CSS_SELECTOR , 'table.CSSTableGenerator .ng-binding')))

html = browser.find_element_by_css_selector('table.CSSTableGenerator')
soup = BeautifulSoup(html.get_attribute("outerHTML"), 'lxml')
print(soup.prettify().encode('utf-8'))