extract text from website python

import requests from bs4 import BeautifulSoup #pip install bs4 url = 'https://www.troyhunt.com/the-773-million-record-collection-1-data-reach/' res = requests.get(url) html_page = res.content soup = BeautifulSoup(html_page, 'html.parser') text = soup.find_all(text=True) output = '' blacklist = [ '[document]', 'noscript', 'header', 'html', 'meta', 'head', 'input', 'script', # there may be more elements you don't want, such as "style", etc. ] for t in text: if t.parent.name not in blacklist: output += '{} '.format(t) print(output)

>>> from bs4 import BeautifulSoup >>> raw_html = open('contrived.html').read() >>> html = BeautifulSoup(raw_html, 'html.parser') >>> for p in html.select('p'): ... if p['id'] == 'walrus': ... print(p.text) 'I am the walrus'

web scraper python

def get_hits_on_name(name):
    """
    Accepts a `name` of a mathematician and returns the number
    of hits that mathematician's Wikipedia page received in the 
    last 60 days, as an `int`
    """
    # url_root is a template string that is used to build a URL.
    url_root = 'URL_REMOVED_SEE_NOTICE_AT_START_OF_ARTICLE'
    response = simple_get(url_root.format(name))

    if response is not None:
        html = BeautifulSoup(response, 'html.parser')

        hit_link = [a for a in html.select('a')
                    if a['href'].find('latest-60') > -1]

        if len(hit_link) > 0:
            # Strip commas
            link_text = hit_link[0].text.replace(',', '')
            try:
                # Convert to integer
                return int(link_text)
            except:
                log_error("couldn't parse {} as an `int`".format(link_text))

    log_error('No pageviews found for {}'.format(name))
    return None

Posted by: Guest on August-09-2020

Source

Code answers related to "extract text from website python"

Code answers related to "Python"

Browse Popular Code Answers by Language

Answers for "extract text from website python"

Code answers related to "extract text from website python"

Code answers related to "Python"

Python Answers by Framework

Browse Popular Code Answers by Language

Popular Programming Languages

Advertisements

Company

Compilers

Help

Connect with us