How Fetch All Data And Parse Using Meta In Scrapy?

I want to save all of the data in a json file. how can i parse my data using meta? I don't know my meta format is ok or not. finally yield the all of the data (which i through by m

Solution 1:

Here is the answer according to your question:

If you want to transfer data from one parse methon to another using meta, you need to create key for each value and injected each key-value pair in Request using meta ,after all, in parse_v method, you have to create key newly and to grab previous key using response.meta and it's the new key-value pairs to yield data like 'Category': response.meta['cat']

classTrendyolSpider(scrapy.Spider):
    name = 'data'
    start_urls = [
        'https://www.trendyol.com/olalook/kadin-siyah-cepli-minik-beyaz-cicekli-klos-elbise-elb-19000480-p-6635101']

    defparse(self, response):
        text = response.xpath(
            "//p/script[contains(@type,'application/ld+json')]/text()").extract_first()
        json_text = json.loads(text)
        items = TrendyolItem()
        products = response.css('div.pd-app-container')
        for product inproducts:
            category = product.css(
                'div.breadcrumb>a:nth-child(3)+ a.breadcrumb-item span::text').get(),
            product_name = product.css('div.pr-in-cn h1.pr-new-br::text').get() + " " + product.css(
                'div.pr-in-cn h1.pr-new-br span::text').get(),
            price = product.css(
                'div.pr-bx-nm  span.prc-org::text').get().replace("TL", ""),
            discount_price = product.css(
                'div.pr-bx-nm  span.prc-slg::text').get().replace("TL", ""),
            brand = response.css("div.sl-nm a::text").get(),
            image = json_text.get('image'),
            size = product.css("div.pr-in-at-sp ::text").getall(),
            product_information = product.css(
                "div.pr-in-dt-cn ::text").getall(),
            product_features = product.css(
                "div.pr-prop-content ::text").getall(),

            items['category'] = category
            items['product_name'] = product_name
            items['price'] = price
            items['discount_price'] = discount_price
            items['brand'] = brand
            items['image'] = image
            items['size'] = size
            items['product_information'] = product_information
            items['product_features'] = product_features

        all_info = response.xpath(
            "//script[contains(@type,'application/javascript')]/text()").extract_first()
        product_json = chompjs.parse_js_object(all_info)
        ides = product_json['product']['productGroupId']

        varient_url = "https://public.trendyol.com/discovery-web-productgw-service/api/productGroup/" + \
            str(ides)

        yield Request(
            url=varient_url,
            callback=self.parse_v,
            meta={'cat': category, 'pro_name': product_name,'p': price, 'dis_price': discount_price,
            'bra': brand,'ima':image,'si':size, 'porduct_info':product_information,'features':product_features
            }
            )

    defparse_v(self, response):
        #items = response.meta['items']
        json_tex5 = json.loads(response.body)
        dataa = json_tex5.get('result').get(
            "slicingAttributes")[0].get("attributes")
        for i indataa:
            all_info = self.start_urls + i['contents'][0]['url'] + "https://cdn.dsmcdn.com"+i['contents'][0]['imageUrl']\
                + i['contents'][0]['price']['discountedPrice']['text'] + \
                i['contents'][0]['price']['originalPrice']['text']

        yield {
            'Category': response.meta['cat'],
            'Product_name': response.meta['pro_name'],
            'Price': response.meta['p'],
            'Discount_price': response.meta['dis_price'],
            'Brand': response.meta['bra'],
            'Image': response.meta['ima'],
            'Size': response.meta['si'],
            'Product_information': response.meta['porduct_info'],
            'Product_features': response.meta['features'],
            'rank': all_info
        }