webscrapping with python

import requests from bs4 import BeautifulSoup URL = 'https://www.monster.com/jobs/search/?q=Software-Developer&where=Australia' page = requests.get(URL) soup = BeautifulSoup(page.content, 'html.parser')

>>> raw_html = simple_get('http://www.fabpedigree.com/james/mathmen.htm') >>> html = BeautifulSoup(raw_html, 'html.parser') >>> for i, li in enumerate(html.select('li')): print(i, li.text) 0 Isaac Newton Archimedes Carl F. Gauss Leonhard Euler Bernhard Riemann 1 Archimedes Carl F. Gauss Leonhard Euler Bernhard Riemann 2 Carl F. Gauss Leonhard Euler Bernhard Riemann 3 Leonhard Euler Bernhard Riemann 4 Bernhard Riemann # 5 ... and many more...

web scraper python

def simple_get(url):
    """
    Attempts to get the content at `url` by making an HTTP GET request.
    If the content-type of response is some kind of HTML/XML, return the
    text content, otherwise return None.
    """
    try:
        with closing(get(url, stream=True)) as resp:
            if is_good_response(resp):
                return resp.content
            else:
                return None

    except RequestException as e:
        log_error('Error during requests to {0} : {1}'.format(url, str(e)))
        return None


def is_good_response(resp):
    """
    Returns True if the response seems to be HTML, False otherwise.
    """
    content_type = resp.headers['Content-Type'].lower()
    return (resp.status_code == 200 
            and content_type is not None 
            and content_type.find('html') > -1)


def log_error(e):
    """
    It is always a good idea to log errors. 
    This function just prints them, but you can
    make it do anything.
    """
    print(e)

Posted by: Guest on August-09-2020

Source

Code answers related to "webscrapping with python"

Code answers related to "Python"

Browse Popular Code Answers by Language

Answers for "webscrapping with python"

Code answers related to "webscrapping with python"

Code answers related to "Python"

Python Answers by Framework

Browse Popular Code Answers by Language

Popular Programming Languages

Advertisements

Company

Compilers

Help

Connect with us