Arkadaşlar merhaba, selenium pyhton ile bir bot yazdım. Bu botta mail atacağım alanlar belli, sadece mail gönder butonuna tıklayıp kullanıcı adı, mail adresi yazmam gönder’ e tıklamam
yeterli içerik zaten kendisinde mevcut. Ancak posta göndereceğim butona tıkladığımda beni giriş yapmaya zorluyor. Doğru kullanıcı bilgilerini yazıyorum ama giriş yapmak istediğimde change validation adı altında 30 saniyelik beklemeye itiyor beni. Bundan sonrada 3 durumdan biri ortaya çıkıyor: (Bot ile değilde kendim aynı bilgileri girdiğimde izin veriyor girişe ama botilegirdiğimde uyarı vermeden girişe izin vermiyor.)
-Sayfa bulunamadı,
-Tekrar girişe atıyor, bu defa da eskimi element uyarısı veriyor,
-Ya da 30 saniye bekler ve hala giriş yapmamış oluyor.
Benim fikrim bot olduğumu farketmiş olmasından kaynaklı olduğunu hissettiriyor. 1 gün içerisinde bunu çözmüş olmam gerektiğinden uzunca yazdım kusura bakmayın. Desteklerinizi bekliyorum.Kodlarımı da ekliyorum. Her bir ilan linkini veritabanından çekiyorum.
import bs4
import urllib.request
from urllib import parse
import pandas as pd
from tkinter import E
import pymysql
import mysql.connector
import configparser
import re
import numpy as np
import time
import concurrent.futures
from icecream import ic
# import erequests
import lxml
from multiprocessing import Pool
# from multiprocessing import Process, Lock
from multiprocessing import Process
from datetime import datetime
from tqdm import tqdm # progress bar
from urllib.request import urlopen
from bs4 import BeautifulSoup
from datetime import datetime
from sqlalchemy import create_engine
from selenium.webdriver.support.ui import Select
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.firefox.options import Options
mydb = mysql.connector.connect(
host="localhost",
user="root",
password="",
database="mail"
)
mycursor = mydb.cursor()
sql = "SELECT ad_link FROM adlinks_de_mail"
mycursor.execute(sql)
myresult = mycursor.fetchall()
all_links = myresult[0:]
len_all_links = len(all_links)
dataframe = pd.DataFrame(all_links, columns=['links'])
x = 0
y = 5
#def fonksiyon(i):
# global x
# global y
number = np.arange(x,y)
for i in tqdm(number):
ad_link = dataframe.links[i] # ad_link = dataframe["links"][i]
ic(ad_link)
fireFoxOptions = Options()
fireFoxOptions.binary_location = r'C:\Program Files\Firefox Developer Edition\firefox.exe'
fireFoxOptions.add_argument('--disable-gpu')
fireFoxOptions.add_argument('--no-sandbox')
driver = webdriver.Firefox(options=fireFoxOptions)
sleep_time = 5
driver.get(ad_link)
time.sleep(sleep_time)
ad_source = driver.page_source
ad_soup = BeautifulSoup(ad_source, 'lxml')
mainresults = ad_soup.find_all('div', {'class': 'cBox cBox--content u-overflow-inherit '})
#class': 'cBox cBox--content u-overflow-inherit
tel_number = ' '
try:
WebDriverWait(driver, timeout=10).until(
lambda d: d.find_element(By.XPATH, "//button[@class='sc-bczRLJ iBneUr mde-consent-accept-btn']")).click()
WebDriverWait(driver, timeout=10).until(
lambda d: d.find_element(By.XPATH, "//p[@class='phone-number-container']")).click()
tel_number = driver.find_element(By.XPATH, "//p[@class='phone-number-container']").text
ic(tel_number)
except:
tel_number = 'Not Found Tel Number'
ic(tel_number)
try:
web_text = driver.find_element(By.XPATH, "/html/body").text()
ic('web_text')
#web_text = driver.find_element(By.XPATH, "/html/body/div[6]/div/div[2]/div[2]/div[1]/div[4]/div[1]").text #"/html/body"
#web_word = driver.find_element(By.XPATH, "/html/body/div[6]/div/div[2]/div[2]/div[1]/div[6]").text
words = ["Import", "Test", "Casd"]
search_words = [word for word in words if re.findall(word, web_text)]
text_words = ''
if search_words:
for i, word in enumerate(search_words):
if i < len(search_words) - 1:
text_words += f"{word}, "
else:
text_words += f"{word}."
ic(f"\nCannot send mail because it contains the word.Index : {text_words}")
ic(re.findall)
except Exception:
text_words = "Not Found Words"
ic(text_words)
time.sleep(1)
try:
brand_and_model = ad_soup.find("h1", {"class": ('h2 u-text-break-word')}).get_text()
except:
brand_and_model = ' '
try:
model_version = ad_soup.find("div", {"class": ('listing-subtitle')}).get_text()
except:
model_version = ' '
try:
location = ad_soup.find("p", {"class": ('seller-address')}).get_text()
except:
location = ' '
try:
url_id = ad_soup.find(" ", {"class": ('')}).get_text()
except:
url_id = ''
cars_data = pd.DataFrame({
'brand_and_model': brand_and_model,
'model_version': model_version,
'location': location,
'tel_number': tel_number,
'url_id': url_id,
},
index=[0])
try:
table_pre = ad_soup.find("div", {"class": "cBox cBox--content cBox-body"}) # 1 (6 in one)
all_div = table_pre.findAll("div", {"class": ('key-feature__content')}) # 6 (2 in one)
all_title = table_pre.findAll("div", {"class": ('key-feature__label')}) # 6
all_results = table_pre.findAll("div", {"class": ('key-feature__value')}) # 6
except:
pass
description_list = []
value_list = []
try:
div_length = len(all_div)
except:
div_length = 6
#ic('all_div')
for i in range(div_length):
try:
description_list.append(all_title[i].text)
description_list = list(map(lambda x: x.replace(" ", "_"), description_list))
value_list.append(all_results[i].text)
except:
description_list.append('')
value_list.append('')
all_key = []
all_value = []
try:
pdiv = ad_soup.find_all('div', {'class': 'bullet-list'})
except:
pass
equipment_key = []
try:
equipment_key_length = len(pdiv)
except:
equipment_key_length = 1
equipment_value = []
try:
dd_ul_li_length = len(pdiv)
except:
dd_ul_li_length = 1
df3 = pd.DataFrame(list(zip(equipment_key, equipment_value)), columns=['all_key', 'all_value'])
df2 = pd.DataFrame(list(zip(all_key, all_value)), columns=['all_key', 'all_value'])
# create a dataframe
df1 = pd.DataFrame(list(zip(description_list, value_list)), columns=['description_list', 'value_list'])
# Sütun adları olarak -description_list- den gelen verileri transpose et
# df = df.T
df1 = df1.set_index('description_list').T.reset_index(drop=True)
df1 = df1.rename_axis(None, axis=1)
# df1['link'] = ad_link
df1.insert(0, "brand_and_model", brand_and_model)
df1.insert(1, "model_version", model_version)
df1.insert(2, "location", location)
df1.insert(5, "tel_number", tel_number)
df2_3 = pd.concat([df2, df3])
df2_3 = df2_3.set_index('all_key').T.reset_index(drop=True)
df2_3 = df2_3.rename_axis(None, axis=1)
df_last = pd.concat([df1, df2_3], axis=1)
df_last = df_last.astype(str).groupby(df_last.columns, sort=False, axis=1).agg(
lambda x: x.apply(','.join, 1))
# datetime string
now = datetime.now()
datetime_string = str(now.strftime("%Y%m%d_%H%M%S"))
df_last['ad_link'] = ad_link
df_last['download_date_time'] = datetime_string
config = configparser.RawConfigParser()
config.read(filenames='my.properties')
scrap_db = pymysql.connect(host='localhost', user='root', password='', database='mail',
charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor)
cursor = scrap_db.cursor()
sql = """CREATE TABLE CARS(
brand_and_model VARCHAR(32),
model_version VARCHAR(64),
location VARCHAR(64),
tel_number VARCHAR(32),
mileage VARCHAR(32),
first_registration DATE(7),
ad_link VARCHAR(256),
download_date_time DATE(32),
search_words VARCHAR(32),
url_id int(9)
)"""
#cursor.execute(sql) #Save data to the table
for row_count in range(0, df_last.shape[0]):
chunk = df_last.iloc[row_count:row_count + 1, :].values.tolist()
brand_and_model = ""
model_version = ""
location = ""
tel_number = ""
mileage = ""
first_registration = ""
ad_link = ""
download_date_time = ""
search_words = ""
url_id = ""
status = ""
# control = "true"
lenght_of_chunk = len(chunk[0])
# ic("lenght_of_chunk:",lenght_of_chunk)
if "brand_and_model" in cars_data:
try:
brand_and_model = chunk[0][0]
except:
brand_and_model = ""
if "model_version" in cars_data:
try:
model_version = chunk[0][1]
except:
model_version = ""
if "location" in cars_data:
try:
location = chunk[0][2]
except:
location = ""
if "tel_number" in cars_data:
try:
tel_number = chunk[0][5]
except:
tel_number = ""
if "Kilometerstand" in description_list:
index_no = description_list.index("Kilometerstand")
try:
mileage = value_list[index_no]
except:
mileage = ""
if "Erstzulassung" in description_list:
index_no = description_list.index("Erstzulassung")
try:
first_registration = value_list[index_no]
except:
first_registration = ""
if chunk[0][lenght_of_chunk - 2] != "":
ad_link = chunk[0][lenght_of_chunk - 2] # ad_link
if chunk[0][lenght_of_chunk - 1] != "":
download_date_time = chunk[0][lenght_of_chunk - 1] # datetime_string
if ((tel_number == 'Not Found Tel Number') and (text_words == 'Not Found Words')):
control = "true"
status = 1
ic('True')
else:
control = "false"
status = 0
ic('False')
time.sleep(5)
if control == "true":
ic("Mail Sending")
bttn = driver.find_element(By.XPATH,'/html/body/div[6]/div/div[2]/div[2]/div[2]/aside/div[1]/div/div[4]/div/span')
bttn.click()
#namePerson = driver.find_element(By.XPATH,'/html/body/div[11]/div/div/div/div/div/div[3]/form/div/div[2]/div[1]/div/div/input')
#driver.maximize_window()
#time.sleep(3)
#driver.find_element(By.XPATH,"/html/body/div[6]/div/div[2]/div[2]/div[2]/aside/div[1]/div/div[4]/div/span").click()
time.sleep(5)
# path = driver.find_element(By.XPATH,"/html/body/div[2]/div[1]/div[1]/div[2]/div")
user = driver.find_element(By.CSS_SELECTOR,"#login-username")
user.click()
time.sleep(3)
user.send_keys("verkauf.bellic@gmail.com")
time.sleep(5)
password = driver.find_element(By.XPATH, "/html/body/div[2]/div[1]/div[1]/div[2]/div/div[1]/form/div[2]/div/input")
password.click()
password.send_keys("W2601.")
button = driver.find_element(By.ID,'login-submit')
button.click()
time.sleep(30)
# path = 1
# textArea = driver.find_element(By.XPATH,'/html/body/div[4]/div/div[2]/div[3]/div[2]/aside/div[1]/div/div[4]/div/div/span').click()
bttn = driver.find_element(By.XPATH,'/html/body/div[6]/div/div[2]/div[2]/div[2]/aside/div[1]/div/div[4]/div/span') # Mail göndereceğimiz ekrandaki mailGönder buton div'i.
bttn.click()
time.sleep(3)
user = driver.find_element(By.CSS_SELECTOR, "#login-username")
user.click()
time.sleep(3)
user.send_keys("v.b@gmail.com")
time.sleep(5)
password = driver.find_element(By.XPATH,"/html/body/div[2]/div[1]/div[1]/div[2]/div/div[1]/form/div[2]/div/input")
password.click()
password.send_keys("W.")
button = driver.find_element(By.ID, 'login-submit')
button.click()
time.sleep(3)
# //*[@id="login-submit"]
if (brand_and_model == ' '):
control = "false"
else:
control = "true"
if control == "true":
mySql_insert_query = "INSERT INTO CARS(brand_and_model,model_version,location,tel_number,mileage,first_registration,ad_link,download_date_time,search_words,url_id,status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
val = (brand_and_model, model_version, location, tel_number, mileage, first_registration, ad_link, download_date_time, search_words, url_id,status)
cursor = scrap_db.cursor()
cursor.execute(mySql_insert_query, val)
scrap_db.commit()
ic(cursor.rowcount, "Record inserted successfully into *CARS* table")
driver.close()
