Uncategorized

How to Extract all Product Data from a Shopify Website using Beautiful Soup

import csv
import requests
from bs4 import BeautifulSoup

# Define the URL and retrieve the page content
url = "USE YOUR OWN URL .ATOM"
response = requests.get(url)
content = response.content

# Create a BeautifulSoup object to parse the page content
soup = BeautifulSoup(content, "xml")

# Extract the data from each <entry> element and store it in a list
products = []
for entry in soup.find_all("entry"):
    #print(entry)
    name = entry.title.text
    #price = entry.svariant.sprice.text
    price_tag = entry.find('s:price')
    price_value = price_tag.text
    price = price_value
    print(name)
    print(price)
    #url = entry.link.get("href")
    #image = entry.summary.find("img").get("src")
    #summary = entry.summary.text.strip()
    currency = "usd"
    price, url, image, summary = "test data", "test data", "test data", "test data"
    products.append((name, price, url, image, currency, summary))

# Save the data to a CSV file
with open("products.csv", "w", newline="") as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(["Name", "Price", "URL", "Image", "Currency", "Summary"])
    for product in products:
        writer.writerow(product)