title: web scraper last pagination with python
body: <p>is it possible to print all the pages on the web, I have problems with pagination, it should print all pages on the web, but it only displays 9 pages, while on the web there are 24 pages,</p>
  <p>I want to print all the pages until the last one, while in pagination there are only 9 lists, whereas on the web there are 24 pages</p>
  <pre><code>import requests
  from bs4 import BeautifulSoup
  
  def login():    
      print('test login')
      urls = &quot;https://www.yelp.com/search?cflt=contractors&amp;find_loc=St%20Francis%20Wood%2C%20San%20Francisco%2C%20CA&amp;start=0&quot;
      headers = {
          'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
          'accept-encoding': 'gzip, deflate, br',
          'accept-language': 'en-US,en;q=0.8',
          'upgrade-insecure-requests': '1',
          'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'
      }
  
      respon = requests.get(urls, headers=headers)
      f = open('./re.html', 'w+')
      f.write(respon.text)
      f.close()
  
      soup = BeautifulSoup(respon.text, 'html5lib')
  
      page_item = soup.find_all('div', attrs={'class': 'pagination-link-container__09f24__13AN7'})
      total_page = len(page_item)
  
      print(len(page_item))
  
      return total_page 
  
  def get_url(page):
      print('test url ...')
      params = {
          'start': page
      }
  
      respon = requests.get('https://www.yelp.com/search?cflt=contractors&amp;find_loc=St%20Francis%20Wood%2C%20San%20Francisco%2C%20CA&amp;start=0', params=params)
      
  
      suop = BeautifulSoup(respon.text, 'html5lib')
  
      titles = suop.find_all('span', attrs={'class': 'text__09f24__2tZKC text-color--black-regular__09f24__1QxyO text-align--left__09f24__3Drs0 text-weight--bold__09f24__WGVdT text-size--inherit__09f24__2rwpp'})
  
      urls = []
      for title in titles:
          url = title.find('a')['href']
          urls.append(url)
  
      return  urls
  
  
  def run():
      total_page = login()
  
      total_urls = []
      for i in range(total_page):
          page = i + 1
  
          # print(page)
          urls =  get_url(page)
          total_urls += urls
      
      # print(total_urls)
      print(len(total_urls))
  
  
  if __name__ == '__main__':
      run()
  </code></pre>