淘宝数据模拟爬取

from selenium import webdriver

from selenium.webdriver.common.by import By

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

from time import sleep

#定义一个taobao类

class taobao_infos:

    #对象初始化

    def __init__(self):

        url = ‘https://login.taobao.com/member/login.jhtml’

        self.url = url

        options = webdriver.ChromeOptions()

        options.add_experimental_option(‘excludeSwitches’, [‘enable-automation’]) # 此步骤很重要，设置为开发者模式，防止被各大网站识别出来使用了Selenium

        self.browser = webdriver.Chrome(options=options)

        self.wait = WebDriverWait(self.browser, 20) #超时时长为10s

     #登录淘宝

    def login(self):

        # 打开网页

        self.browser.get(self.url)

        # 等待 密码登录选项 出现

        password_login = self.wait.until(EC.presence_of_element_located((By.ID, ‘fm-login-password’)))

        self.browser.find_element_by_xpath(‘/html/body/div/div[2]/div[3]/div/div/div/div[2]/div/form/div[5]/a[2]’).click()

        sleep(10)

        taobao_name = self.wait.until(EC.presence_of_element_located((By.XPATH, ‘/html/body/div[1]/div/ul[1]/li[1]/div[1]/div[2]/a’)))

        print(taobao_name.text)

        self.browser.find_element_by_xpath(‘/html/body/div[1]/div/ul[2]/li[1]/div/a/span’).click()

        #等待搜索框加载完毕

        input_search=self.wait.until(EC.presence_of_element_located((By.ID, ‘q’)))

        input_search.clear()

        input_search.send_keys(“华为手机”)

        sleep(1)

        sosuo=self.wait.until(EC.presence_of_element_located((By.XPATH, ‘/html/body/div[2]/div/div/div[2]/div/div[1]/div[2]/form/div[1]/button’)))

        sosuo.click()

        sleep(1)

        #获取商品总数

    def get_goodnumber(self):

        good_number=self.wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,’#mainsrp-pager > div > div > div > div.total’)))

        good_number=good_number.text.replace(“共”,”).replace(“页”,”).replace(“，”,”).strip()

        return good_number

        #开始爬取

    def crawl_goods(self,good_number):

        for page in range(1):

            good_items=self.browser.find_elements_by_class_name(‘J_MouserOnverReq’)

            for good_item in good_items:

                price=good_item.find_element_by_class_name(‘price’).text

                nick_name=good_item.find_element_by_class_name(‘shop’).text

                address=good_item.find_element_by_class_name(‘location’).text

            self.swipe_down(2)

    #模拟向下滑动

    def swipe_down(self,second):

         for i in range(int(second/0.1)):

             js = “var q=document.documentElement.scrollTop=” + str(300+200*i)

             self.browser.execute_script(js)

             sleep(1)

             js = “var q=document.documentElement.scrollTop=100000”

             self.browser.execute_script(js)

             sleep(1)

if __name__ == “__main__”:

    a = taobao_infos()

    a.login() #登录

    numbers=a.get_goodnumber()

    a.crawl_goods(numbers)