04 Scrapy Most used commands

1

Base use of parse() in Spider 

import scrapy


class BookspiderSpider(scrapy.Spider):
    name = "bookspider"
    allowed_domains = ["books.toscrape.com"]
    start_urls = ["https://books.toscrape.com"]

    def parse(self, response):
        books = response.css('article.product_pod')
        for book in books:
            yield{
                'name': book.css('h3 a::text').get(),
                'price': book.css('.product_price .price_color::text').get(),
                'url': book.css('h3 a').attrib['href']
            }

2

,

Be sure, change in spider directory, /bookscraper/bookscraper/spider

run this, the last argument is the domain url

scrapy genspider bookspider books.toscrape.com

3

Find html data to get in scrapy shell, then add to parse() yeild

scrapy shell
fetch('https://books.toscrape.com/')
book.css('h3 a').attrib['href']
response.css('li.next a::attr(href)').get()
scrapy crawl bookspider

4.

This only gets 40 results, because next_page sometimes includes 'catalogue' 

import scrapy
class BookspiderSpider(scrapy.Spider):
    name = "bookspider"
    allowed_domains = ["books.toscrape.com"]
    start_urls = ["https://books.toscrape.com"]

    def parse(self, response):
        books = response.css('article.product_pod')
        for book in books:
            yield{
                'name': book.css('h3 a::text').get(),
                'price': book.css('.product_price .price_color::text').get(),
                'url': book.css('h3 a').attrib['href']
            }
        next_page = response.css('li.next a::attr(href)').get()

        if next_page is not None:
            next_page_url = 'https://books.toscrape.com/' + next_page
            yield response.follow(next_page_url,callback=self.parse)

3

fix url follow() with condition of with or without catalogue 

import scrapy
class BookspiderSpider(scrapy.Spider):
    name = "bookspider"
    allowed_domains = ["books.toscrape.com"]
    start_urls = ["https://books.toscrape.com"]

    def parse(self, response):
        books = response.css('article.product_pod')
        for book in books:
            yield{
                'name': book.css('h3 a::text').get(),
                'price': book.css('.product_price .price_color::text').get(),
                'url': book.css('h3 a').attrib['href']
            }
        next_page = response.css('li.next a::attr(href)').get()
        if next_page is not None:
            if 'catalogue/' in next_page:
                next_page_url = 'https://books.toscrape.com/' + next_page
            else:
                next_page_url = 'https://books.toscrape.com/catalogue/' + next_page
            yield response.follow(next_page_url,callback=self.parse)

The item_scraped_count shows 1000 now