Merhaba, bir siteden topladığım ilan linklerini tek tek çekerek mail göndermek istiyorum. Bir sistem yazdım ancak çalışmıyor, hata da vermiyor. Şöyle ki ilanda belirli kelimeler ve telefon numarası var ise mail atmayacak, yoksa mail atacak. Mail botu kendi başına çalışıyor. Ancak filtrelerle birleştirince olmuyor.Mail atsa da atmasada marka, model, location, tel(varsa), km… gibi bilgileri veritabanına yazması gerekiyor. Kodlarımı paylaşıyorum. Baktığınızda gördüğünüz hatalar var ise (mantık, kod hatası gibi…) paylaşırsanız çok sevinirim.
mydb = mysql.connector.connect(
host="localhost", user="root", password="", database="mail"
)
mycursor = mydb.cursor()
sql = "SELECT ad_link FROM adlink"
mycursor.execute(sql)
myresult = mycursor.fetchall()
all_links = myresult[0:]
len_all_links = len(all_links)
dataframe = pd.DataFrame(all_links, columns=["links"])
x = 0
y = 10
def fonksiyon(i):
global x
global y
number = np.arange(x, y)
ad_link = dataframe.links[i] # ad_link = dataframe["links"][i]
print(ad_link)
fireFoxOptions = Options()
fireFoxOptions.binary_location = (
r"C:\Program Files\Firefox Developer Edition\firefox.exe"
)
fireFoxOptions.add_argument("--disable-gpu")
fireFoxOptions.add_argument("--no-sandbox")
driver = webdriver.Firefox(options=fireFoxOptions)
sleep_time = 5
driver.get(ad_link)
time.sleep(sleep_time)
ad_source = driver.page_source
ad_soup = BeautifulSoup(ad_source, "lxml")
mainresults = ad_soup.find_all(
"div", {"class": "cBox cBox--content u-overflow-inherit "}
)
try:
brand_and_model = ad_soup.find(
"h1", {"class": ("h2 u-text-break-word")}
).get_text()
except:
brand_and_model = " "
try:
model_version = ad_soup.find("div", {"class": ("listing-subtitle")}).get_text()
except:
model_version = " "
try:
location = ad_soup.find("p", {"class": ("seller-address")}).get_text()
except:
location = " "
try:
WebDriverWait(driver, timeout=10).until(
lambda d: d.find_element(
By.XPATH, "//button[@class='sc-bczRLJ iBneUr mde-consent-accept-btn']"
)
).click()
WebDriverWait(driver, timeout=10).until(
lambda d: d.find_element(By.XPATH, "//p[@class='phone-number-container']")
).click()
tel_number = driver.find_element(
By.XPATH, "//p[@class='phone-number-container']"
).text
# print(tel_number)
except:
tel_number = " "
# print("NOT found tel number")
try:
contact_name = ad_soup.find(
"div", {"class": ("g-col-10 u-no-pad-left")}
).get_text()
except:
contact_name = " "
try:
re = driver.find_element(By.XPATH, "/html/body")
search_words = re.findAll(
"Import",
"Mail",
)
print("\nkelimesini içerdiği için mail gönderilemiyor.Indeksi :", i)
print(re.findAll)
except:
search_words = " "
time.sleep(1)
cars_data = pd.DataFrame(
{
"brand_and_model": brand_and_model,
"model_version": model_version,
"location": location,
"tel_number": tel_number,
"contact_name": contact_name,
"search_words": search_words,
#"url_id": url_id,
},
index=[0],
)
try:
table_pre = ad_soup.find(
"div", {"class": "cBox cBox--content cBox-body"}
) # 1 (6 in one)
all_div = table_pre.findAll(
"div", {"class": ("key-feature__content")}
) # 6 (2 in one)
all_title = table_pre.findAll("div", {"class": ("key-feature__label")}) # 6
all_results = table_pre.findAll("div", {"class": ("key-feature__value")}) # 6
except:
pass
description_list = []
value_list = []
try:
div_length = len(all_div)
except:
div_length = 6
for i in range(div_length):
try:
description_list.append(all_title[i].text)
description_list = list(
map(lambda x: x.replace(" ", "_"), description_list)
)
value_list.append(all_results[i].text)
except:
description_list.append("")
value_list.append("")
all_key = []
all_value = []
try:
pdiv = ad_soup.find_all("div", {"class": "bullet-list"})
except:
pass
equipment_keys = "Features"
equipment_key = []
try:
equipment_key_length = len(pdiv)
except:
equipment_key_length = 1
for k in range(equipment_key_length):
try:
equipment_key.append(equipment_keys)
except:
equipment_key.append("")
equipment_value = []
try:
dd_ul_li_length = len(pdiv)
except:
dd_ul_li_length = 1
df3 = pd.DataFrame(
list(zip(equipment_key, equipment_value)), columns=["all_key", "all_value"]
)
df2 = pd.DataFrame(list(zip(all_key, all_value)), columns=["all_key", "all_value"])
df1 = pd.DataFrame(
list(zip(description_list, value_list)),
columns=["description_list", "value_list"],
)
df1 = df1.set_index("description_list").T.reset_index(drop=True)
df1 = df1.rename_axis(None, axis=1)
df1.insert(0, "brand_and_model", brand_and_model)
df1.insert(1, "model_version", model_version)
df1.insert(2, "location", location)
df1.insert(5, "tel_number", tel_number)
df2_3 = pd.concat([df2, df3])
df2_3 = df2_3.set_index("all_key").T.reset_index(drop=True)
df2_3 = df2_3.rename_axis(None, axis=1)
df_last = pd.concat([df1, df2_3], axis=1)
df_last = (
df_last.astype(str)
.groupby(df_last.columns, sort=False, axis=1)
.agg(lambda x: x.apply(",".join, 1))
)
now = datetime.now()
datetime_string = str(now.strftime("%Y%m%d_%H%M%S"))
df_last["ad_link"] = ad_link
df_last["download_date_time"] = datetime_string
config = configparser.RawConfigParser()
config.read(filenames="my.properties")
scrap_db = pymysql.connect(
host="localhost",
user="root",
password="",
database="mail",
charset="utf8mb4",
cursorclass=pymysql.cursors.DictCursor,
)
cursor = scrap_db.cursor()
sql = """CREATE TABLE CARS(
brand_and_model VARCHAR(32),
model_version VARCHAR(64),
location VARCHAR(64),
tel_number VARCHAR(32),
mileage VARCHAR(32),
ts VARCHAR(7),
ad_link VARCHAR(256),
download_date_time VARCHAR(32),
search_words VARCHAR(32),
url_id int(9)
)"""
cursor.execute(sql)
# MAİL GÖNDERİMİ
for row_count in range(0, df_last.shape[0]):
chunk = df_last.iloc[row_count : row_count + 1, :].values.tolist()
brand_and_model = ""
model_version = ""
location = ""
tel_number = ""
mileage = ""
ts = ""
ad_link = ""
download_date_time = ""
search_words = ""
url_id = ""
lenght_of_chunk = len(chunk[0])
if "brand_and_model" in cars_data:
try:
brand_and_model = chunk[0][0]
except:
brand_and_model = ""
if "model_version" in cars_data:
try:
model_version = chunk[0][1]
except:
model_version = ""
if "location" in cars_data:
try:
location = chunk[0][2]
except:
location = ""
if "tel_number" in cars_data:
try:
tel_number = chunk[0][5]
except:
tel_number = ""
if "Kilometerstand" in description_list:
index_no = description_list.index("Kilometerstand")
try:
mileage = value_list[index_no]
except:
mileage = ""
if "ts" in description_list:
index_no = description_list.index("ts")
try:
ts = value_list[index_no]
except:
ts = ""
if chunk[0][lenght_of_chunk - 2] != "":
ad_link = chunk[0][lenght_of_chunk - 2] # ad_link
if chunk[0][lenght_of_chunk - 1] != "":
download_date_time = chunk[0][lenght_of_chunk - 1] # datetime_string
if brand_and_model == " ":
control = "false"
else:
control = "true"
if control == "true":
mySql_insert_query = "INSERT INTO CARS (brand_and_model,model_version,location,tel_number,mileage,ts,ad_link,download_date_time,url_id) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)"
val = (
brand_and_model,
model_version,
location,
tel_number,
contact_name,
mileage,
ts,
ad_link,
download_date_time,
url_id,
)
# cursor = scrap_db.cursor()
cursor.execute(mySql_insert_query, val)
# cursor.executemany(mySql_insert_query, tuple_of_tuples)
scrap_db.commit()
print(cursor.rowcount, "Record inserted successfully into *CARS* table")
# Disconnect from server
# scrap_db.close()
if tel_number == "" and search_words == " ":
print("Mail Sending")
namePerson = driver.find_element(By.XPATH,"/html/body/div[11]/div/div/div/div/div/div[3]/form/div/div[2]/div[1]/div/div/input" )
driver.maximize_window()
time.sleep(3)
driver.find_element(By.XPATH, "/html/body/div[18]/div[2]/div[2]/div[1]/button" ).click()
time.sleep(5)
textArea = driver.find_element(By.XPATH, "/html/body/div[4]//div[2]/aside/div[1]/div/div[4]/div/div/span" ).click()
time.sleep(3)
namePerson = driver.find_element(By.XPATH,"/html/body/div[11]/div//div/div[3]/form/div/div[2]/div[1]/div/div/input")
time.sleep(5)
namePerson.click()
namePerson.send_keys("Kurt")
# namePerson.send_keys(Keys.RETURN)
time.sleep(3)
mailPerson = driver.find_element(By.XPATH,"/html/body/div[11]/div/div/div[3]/form/div/div[2]/div[5]/div/div/input" )
mailPerson.click()
time.sleep(5)
mailPerson.send_keys("test@gmail.com")
time.sleep(3)
sendButton = driver.find_element(By.ID, "vip-contact-form-submit").click()
time.sleep(5)
else:
pass
driver.close()
if __name__ == "__main__":
with concurrent.futures.ProcessPoolExecutor() as executor: # ThreadPoolExecutor
i = list(range(x, y)) # i = [0,1,2,3...100]
executor.map(fonksiyon, i)