create web scrapper in python

import scrapy from ..items import SampletestItem #items class class QuoteTestSpider(scrapy.Spider): name = 'quote_test' start_urls = ['https://quotes.toscrape.com/'] def parse(self, response): items = SampletestItem() #items class quotes = response.css("div.quote") for quote in quotes: items['title'] = quote.css("span.text::text").get() items['author'] = quote.css(".author::text").get() items['tags'] = quote.css(".tags .tag::text").getall() yield items next_page = response.css(".next a::attr(href)").get() if next_page is not None: next_url = response.urljoin(next_page) yield scrapy.Request(next_url, callback=self.parse)

web scraper python

def simple_get(url):
    """
    Attempts to get the content at `url` by making an HTTP GET request.
    If the content-type of response is some kind of HTML/XML, return the
    text content, otherwise return None.
    """
    try:
        with closing(get(url, stream=True)) as resp:
            if is_good_response(resp):
                return resp.content
            else:
                return None

    except RequestException as e:
        log_error('Error during requests to {0} : {1}'.format(url, str(e)))
        return None


def is_good_response(resp):
    """
    Returns True if the response seems to be HTML, False otherwise.
    """
    content_type = resp.headers['Content-Type'].lower()
    return (resp.status_code == 200 
            and content_type is not None 
            and content_type.find('html') > -1)


def log_error(e):
    """
    It is always a good idea to log errors. 
    This function just prints them, but you can
    make it do anything.
    """
    print(e)

Posted by: Guest on August-09-2020

Source

Code answers related to "create web scrapper in python"

Code answers related to "Python"

Browse Popular Code Answers by Language

Answers for "create web scrapper in python"

Code answers related to "create web scrapper in python"

Code answers related to "Python"

Python Answers by Framework

Browse Popular Code Answers by Language

Popular Programming Languages

Advertisements

Company

Compilers

Help

Connect with us