Merhaba aşağıda yazdığım projede yer alan 'updateSql = "UPDATE part2 SET
status= ‘0’ WHERE id = " + str(partId), sorgusu veritabanı şiştiği için çalışmıyor. Kendi başına gayet sağlıklı çalışıyor. Ancak proje içerisinde çalışmıyor. Veritabanını yoran ve sorgunun çalışmasını engelleyen neresidir, nasıl çözüm sağlayabilirm?
mydb = mysql.connector.connect(
host="localhost",
user="root",
password="",
database="tel_number"
)
mycursor = mydb.cursor()
sql = "SELECT ad_link,id FROM part2 where status = 1"
mycursor.execute(sql)
myresult = mycursor.fetchall()
x = 0
y = 10
for row in myresult:
ad_link = row[0]
partId = row[1]
updateSql = "UPDATE part2 SET `status` = '0' WHERE id = " + str(partId)
mycursor.execute(updateSql)
fireFoxOptions = Options()
fireFoxOptions.binary_location = r'C:\Program Files\Firefox Developer Edition\firefox.exe'
fireFoxOptions.add_argument("--headless")
fireFoxOptions.add_argument("--window-size=1920,1080")
fireFoxOptions.add_argument('--start-maximized')
fireFoxOptions.add_argument('--disable-gpu')
fireFoxOptions.add_argument('--no-sandbox')
driver = webdriver.Firefox(options=fireFoxOptions)
sleep_time = 5
driver.get(ad_link)
time.sleep(sleep_time)
ad_source = driver.page_source
ad_soup = BeautifulSoup(ad_source, 'html.parser') # html.parser
mainresults = ad_soup.find_all('div', {'class': 'cBox cBox--content u-overflow-inherit '})
try:
brand_and_model = ad_soup.find("h1", {"class": ('h2 u-text-break-word')}).get_text()
except:
brand_and_model = ' '
try:
model_version = ad_soup.find("div", {"class": ('listing-subtitle')}).get_text()
except:
model_version = ' '
try:
location = ad_soup.find("p", {"class": ('seller-address')}).get_text()
except:
location = ' '
try:
price = ad_soup.find("span", {"class": ('h3')}).get_text()
except:
price = ' '
time.sleep(5)
try:
WebDriverWait(driver, timeout=10).until(
lambda d: d.find_element(By.XPATH, "//button[@class='sc-bczRLJ iBneUr mde-consent-accept-btn']")).click()
WebDriverWait(driver, timeout=10).until(
lambda d: d.find_element(By.XPATH, "//p[@class='phone-number-container']")).click()
tel_number = driver.find_element(By.XPATH, "//p[@class='phone-number-container']").text
except:
tel_number = ' '
try:
rating = ad_soup.find("div", {"class": ('mde-price-rating__badge__label')}).get_text()
except:
rating = ''
print(rating)
cars_data = pd.DataFrame({
'brand_and_model': brand_and_model,
'model_version': model_version,
'location': location,
'price': price,
'tel_number': tel_number,
'rating': rating,
},
index=[0])
try:
table_pre = ad_soup.find("div", {"class": "cBox cBox--content cBox-body"}) # 1 (6 in one)
all_div = table_pre.findAll("div", {"class": ('key-feature__content')}) # 6 (2 in one)
all_title = table_pre.findAll("div", {"class": ('key-feature__label')}) # 6
all_results = table_pre.findAll("div", {"class": ('key-feature__value')}) # 6
except:
pass
description_list = []
value_list = []
try:
div_length = len(all_div)
except:
div_length = 6
for i in range(div_length):
try:
description_list.append(all_title[i].text)
description_list = list(map(lambda x: x.replace(" ", "_"), description_list))
value_list.append(all_results[i].text)
except:
description_list.append('')
value_list.append('')
all_key = []
all_value = []
try:
pdiv = ad_soup.find_all('div', {'class': 'bullet-list'})
except:
pass
equipment_keys = "Features"
equipment_key = []
try:
equipment_key_length = len(pdiv)
except:
equipment_key_length = 1
for k in range(equipment_key_length):
try:
equipment_key.append(equipment_keys)
except:
equipment_key.append('')
try:
pdiv = ad_soup.find_all('div', {'class': 'bullet-list'})
equipment_values = pdiv
except:
pass
equipment_value = []
try:
dd_ul_li_length = len(pdiv)
except:
dd_ul_li_length = 1
df3 = pd.DataFrame(list(zip(equipment_key, equipment_value)), columns=['all_key', 'all_value'])
df2 = pd.DataFrame(list(zip(all_key, all_value)), columns=['all_key', 'all_value'])
df1 = pd.DataFrame(list(zip(description_list, value_list)), columns=['description_list', 'value_list'])
df1 = df1.set_index('description_list').T.reset_index(drop=True)
df1 = df1.rename_axis(None, axis=1)
df1.insert(0, "brand_and_model", brand_and_model)
df1.insert(1, "model_version", model_version)
df1.insert(2, "location", location)
df1.insert(3, "price", price)
df1.insert(4, "tel_number", tel_number)
df1.insert(5, "rating", rating)
df2_3 = pd.concat([df2, df3])
df2_3 = df2_3.set_index('all_key').T.reset_index(drop=True)
df2_3 = df2_3.rename_axis(None, axis=1)
df_last = pd.concat([df1, df2_3], axis=1)
df_last = df_last.astype(str).groupby(df_last.columns, sort=False, axis=1).agg(
lambda x: x.apply(','.join, 1))
#datetime string
now = datetime.now()
datetime_string = str(now.strftime("%Y%m%d_%H%M%S"))
try:
Vehicle_Description = ad_soup.find("div", {"class": ('g-col-12 description')}).get_text()
except:
Vehicle_Description = ' '
df_last['ad_link'] = ad_link
df_last['download_date_time'] = datetime_string
config = configparser.RawConfigParser()
config.read(filenames='my.properties')
# print(config.sections())
scrap_db = pymysql.connect(host='localhost', user='root', password='', database='tel_number',
charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor)
cursor = scrap_db.cursor()
for row_count in range(0, df_last.shape[0]):
chunk = df_last.iloc[row_count:row_count + 1, :].values.tolist()
brand_and_model = ""
model_version = ""
carlist_id = ""
location = ""
price = ""
tel_number = ""
rating = ""
mileage = ""
first_registration = ""
vehicle_description = ""
ad_link = ""
download_date_time = ""
rating = ""
lenght_of_chunk = len(chunk[0])
if "brand_and_model" in cars_data:
try:
brand_and_model = chunk[0][0]
except:
brand_and_model = ""
if "model_version" in cars_data:
try:
model_version = chunk[0][1]
except:
model_version = ""
if "location" in cars_data:
try:
location = chunk[0][2]
except:
location = ""
if "price" in cars_data:
try:
price = chunk[0][3]
except:
price = ""
if "tel_number" in cars_data:
try:
tel_number = chunk[0][4]
except:
tel_number = ""
if "rating" in cars_data:
try:
rating = chunk[0][5]
except:
rating = ""
if "Kilometerstand" in description_list:
index_no = description_list.index("Kilometerstand")
try:
mileage = value_list[index_no]
except:
mileage = ""
if "Erstzulassung" in description_list:
index_no = description_list.index("Erstzulassung")
try:
first_registration = value_list[index_no]
except:
first_registration = ""
if chunk[0][lenght_of_chunk - 4] != "":
vehicle_description = chunk[0][lenght_of_chunk - 4]
if chunk[0][lenght_of_chunk - 2] != "":
ad_link = chunk[0][lenght_of_chunk - 2] # ad_link
if chunk[0][lenght_of_chunk - 1] != "":
download_date_time = chunk[0][lenght_of_chunk - 1]
mySql_insert_query = "INSERT INTO tel_number (brand_and_model,model_version,carlist_id,location,price,tel_number,mileage,first_registration,vehicle_description,ad_link,download_date_time,rating) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
val = (brand_and_model, model_version, carlist_id, location, price, tel_number, mileage, first_registration,
vehicle_description, ad_link, download_date_time, rating)
# cursor = scrap_db.cursor()
cursor.execute(mySql_insert_query, val) # cursor.executemany(mySql_insert_query, tuple_of_tuples)
scrap_db.commit()
print(cursor.rowcount, "Record inserted successfully into *tel_number* table")
url = "https://binfo/api/v1/extension.php"
payload = json.dumps({
"page": "addPhone",
"username": "a",
"password": "UaeiDdZZw86pR1UbTZul2VzM+oxlmRjkKKRmMePrwmyEwIaF9x2WIjjYFIa2aRFeA==",
"country": "test",
"phone": tel_number,
"ad_id": "358649056",
"check_url": ad_link,
"domain": "m.de",
"ad_title": brand_and_model,
"ad_price": price,
"ad_price_rating": rating,
"registry_date": "01/" + first_registration,
"km": mileage,
"description": vehicle_description
})
headers = {
'Content-Type': 'application/json'
}
response = requests.request("POST", url, headers=headers, data=payload)
ic(response.status_code)
ic(response.text)
while 1:
time.sleep(1800)
os.system("start python main.py")
ic('in progress')