mirror of
https://git.auk.su/Dinect/bonus-import-tools.git
synced 2025-12-28 18:10:01 +00:00
added docker-compose.yaml and Dockerfile
This commit is contained in:
Sergey
319
app.py
319
app.py
@@ -1,140 +1,247 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
# __author__ = 'szhdanoff@gmail.com'
|
||||
__version__ = '1.0.1'
|
||||
import os
|
||||
import time
|
||||
import csv
|
||||
import phonenumbers
|
||||
|
||||
import apscheduler
|
||||
#
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from email_validator import validate_email, EmailNotValidError
|
||||
from dotenv import load_dotenv
|
||||
# local imports
|
||||
from dinect_api import get_user
|
||||
from dinect_api import new_user
|
||||
from dinect_api import new_user_by_card
|
||||
from dinect_api import bonuses_update
|
||||
|
||||
load_dotenv()
|
||||
|
||||
is_prod = bool(os.getenv('PRODUCTION', False))
|
||||
COUNTRY = os.getenv('COUNTRY', 'RU')
|
||||
|
||||
DRY_RUN = bool(os.getenv('DRY_RUN', False))
|
||||
|
||||
csv_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'csv')
|
||||
|
||||
files = []
|
||||
# r=root, d=directories, f = files
|
||||
for r, d, f in os.walk(csv_path):
|
||||
for file in f:
|
||||
filename, file_extension = os.path.splitext(file)
|
||||
if file_extension == '.csv':
|
||||
files.append(os.path.join(r, file))
|
||||
print('Script version:', __version__, '| CSV path:', csv_path, '| COUNTRY:', COUNTRY, '| DRY_RUN:', DRY_RUN)
|
||||
|
||||
for f in files:
|
||||
with open(f + f'-{time.strftime("%Y%m%d-%H%M%S", time.localtime())}.log', "w", encoding="utf-8") as log_file:
|
||||
with open(f, "r", encoding="utf-8") as csv_file:
|
||||
file_name = os.path.basename(f)
|
||||
# USERS file
|
||||
if 'users' in file_name:
|
||||
print(f'Processing "users" file: {f}')
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
line_count = 0
|
||||
for row in csv_reader:
|
||||
if line_count == 0:
|
||||
line_count += 1
|
||||
else:
|
||||
try:
|
||||
print(f'Processing line {line_count}: {row}')
|
||||
nickname, full_name, card, phone, email, gender = row
|
||||
# strip whitespaces
|
||||
nickname = nickname.strip()
|
||||
full_name = full_name.strip()
|
||||
card = card.strip()
|
||||
# validate phone
|
||||
phone = phone.strip()
|
||||
try:
|
||||
parsed_phone = phonenumbers.parse(phone, region=COUNTRY)
|
||||
if phonenumbers.is_valid_number(parsed_phone):
|
||||
phone = phonenumbers.format_number(parsed_phone, phonenumbers.PhoneNumberFormat.E164)
|
||||
except:
|
||||
print(f'error in line: [{line_count}]- Invalid phone number: {phone}')
|
||||
log_file.write(f'error in line: [{line_count}]- Invalid phone number: {phone}\n')
|
||||
# validate email
|
||||
email = email.strip()
|
||||
try:
|
||||
email_info = validate_email(email, check_deliverability=False)
|
||||
email = email_info.normalized
|
||||
except EmailNotValidError as e:
|
||||
print(f'error in line: [{line_count}]- Invalid email: {email}')
|
||||
log_file.write(f'error in line: [{line_count}]- Invalid email: {email}\n')
|
||||
email = None
|
||||
# set email if not found
|
||||
if email is None:
|
||||
email = f'{card}@user.dinect.com'
|
||||
|
||||
# validate / set gender
|
||||
gender = gender.strip()
|
||||
if gender not in ['M', 'F']:
|
||||
gender = 1
|
||||
else:
|
||||
if gender == 'M':
|
||||
def validate_phone(raw_phone):
|
||||
"""
|
||||
Validates a given phone number.
|
||||
|
||||
Args:
|
||||
raw_phone (str): The phone number to be validated.
|
||||
|
||||
Returns:
|
||||
str: The validated phone number in E164 format if valid, otherwise an empty string.
|
||||
"""
|
||||
raw_phone = raw_phone.strip()
|
||||
try:
|
||||
raw_phone = phonenumbers.parse(raw_phone, COUNTRY)
|
||||
if phonenumbers.is_valid_number(raw_phone):
|
||||
return phonenumbers.format_number(raw_phone, phonenumbers.PhoneNumberFormat.E164)
|
||||
else:
|
||||
return ''
|
||||
except phonenumbers.NumberParseException:
|
||||
return ''
|
||||
|
||||
|
||||
def run_import():
|
||||
"""
|
||||
Imports users and transactions data from CSV files, validates the data,
|
||||
creates new users if necessary, and updates user bonuses.
|
||||
|
||||
Parameters:
|
||||
None
|
||||
|
||||
Returns:
|
||||
None
|
||||
"""
|
||||
files = []
|
||||
# r=root, d=directories, f = files
|
||||
for r, d, f in os.walk(csv_path):
|
||||
for file in f:
|
||||
filename, file_extension = os.path.splitext(file)
|
||||
if file_extension == '.csv':
|
||||
files.append(os.path.join(r, file))
|
||||
|
||||
for f in files:
|
||||
with open(f + f'-{time.strftime("%Y%m%d-%H%M%S", time.localtime())}.log', "w", encoding="utf-8") as log_file:
|
||||
with open(f, "r", encoding="utf-8") as csv_file:
|
||||
file_name = os.path.basename(f)
|
||||
# USERS file
|
||||
start_time = time.time()
|
||||
if 'users' in file_name:
|
||||
print(f'Processing "users" file: {f}')
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
line_count = 0
|
||||
for row in csv_reader:
|
||||
if line_count == 0:
|
||||
line_count += 1
|
||||
else:
|
||||
line_count += 1
|
||||
try:
|
||||
print(f'Processing line {line_count}: {row}')
|
||||
nickname, full_name, card, phone, email, gender = row
|
||||
|
||||
# strip whitespaces
|
||||
nickname = nickname.strip()
|
||||
full_name = full_name.strip()
|
||||
card = card.strip()
|
||||
|
||||
# validate phone
|
||||
phone = validate_phone(phone)
|
||||
if phone == '':
|
||||
print(f'error in line: [{line_count}]- Invalid phone number: {phone}')
|
||||
log_file.write(f'error in line: [{line_count}]- Invalid phone number: {phone}\n')
|
||||
|
||||
# validate email
|
||||
email = email.strip()
|
||||
try:
|
||||
email_info = validate_email(email, check_deliverability=False)
|
||||
email = email_info.normalized
|
||||
except EmailNotValidError as e:
|
||||
print(f'error in line: [{line_count}]- Invalid email: {email}')
|
||||
log_file.write(f'error in line: [{line_count}]- Invalid email: {email}\n')
|
||||
email = None
|
||||
# set email if not found
|
||||
if email is None:
|
||||
email = f'{card}@user.dinect.com'
|
||||
|
||||
# validate / set gender
|
||||
gender = gender.strip()
|
||||
if gender not in ['M', 'F']:
|
||||
gender = 1
|
||||
else:
|
||||
gender = 2
|
||||
if gender == 'M':
|
||||
gender = 1
|
||||
else:
|
||||
gender = 2
|
||||
|
||||
line_count += 1
|
||||
except ValueError as e:
|
||||
ret = f'Unexpected error in line: [{line_count}] {repr(e)}'
|
||||
log_file.write(f'{ret}\n')
|
||||
except ValueError as e:
|
||||
ret = f'Unexpected error in line: [{line_count}] {repr(e)}'
|
||||
log_file.write(f'{ret}\n')
|
||||
|
||||
# Find user via the API
|
||||
# by card
|
||||
user_found_by_card, user_id, user_card, purchases_url, data = get_user(card)
|
||||
print(f'user_found_by_card {card}', user_found_by_card)
|
||||
# by phone
|
||||
phone = phone.replace('+', '')
|
||||
if not user_found_by_card:
|
||||
user_found_by_phone, user_id, user_card, purchases_url, data = get_user(phone, get_type='phone')
|
||||
print(f'user_found_by_phone {phone}', user_found_by_phone)
|
||||
# Find user via the API
|
||||
# by card
|
||||
user_found_by_card, user_id, user_card, purchases_url, data = get_user(card)
|
||||
print(f'user_found_by_card {card}', user_found_by_card)
|
||||
# by phone
|
||||
phone = phone.replace('+', '')
|
||||
if not user_found_by_card:
|
||||
user_found_by_phone, user_id, user_card, purchases_url, data = get_user(phone,
|
||||
get_type='phone')
|
||||
print(f'user_found_by_phone {phone}', user_found_by_phone)
|
||||
|
||||
user_found = user_found_by_card or user_found_by_phone
|
||||
user_found = user_found_by_card or user_found_by_phone
|
||||
|
||||
# create new user if not found
|
||||
if not user_found:
|
||||
# user_created, data = new_user(
|
||||
# full_name=nickname, phone=phone, gender=1, foreign_card=None, email=email,
|
||||
# )
|
||||
user_created, data = new_user_by_card(
|
||||
full_name=nickname, phone=phone, gender=1, external_card=card, email=email, use_existing=False
|
||||
)
|
||||
if user_created:
|
||||
print('User created with', data['ID'], data['DIN'])
|
||||
# create new user if not found
|
||||
if not user_found:
|
||||
# user_created, data = new_user(
|
||||
# full_name=nickname, phone=phone, gender=1, foreign_card=None, email=email,
|
||||
# )
|
||||
user_created, data = new_user_by_card(
|
||||
full_name=nickname, phone=phone, gender=1, external_card=card, email=email,
|
||||
use_existing=False
|
||||
)
|
||||
if user_created:
|
||||
print('User created with', data['ID'], data['DIN'])
|
||||
else:
|
||||
log_file.write(f'error in line: [{line_count}]- Invalid user data: {data}\n')
|
||||
else:
|
||||
log_file.write(f'error in line: [{line_count}]- Invalid user data: {data}\n')
|
||||
else:
|
||||
print('User found with', user_id, user_card, purchases_url)
|
||||
print('User found with', data['DIN'], user_card)
|
||||
|
||||
end_time = time.time()
|
||||
print(f'Elapsed time of USERS file processing : {end_time - start_time} seconds')
|
||||
|
||||
|
||||
|
||||
# TRANSACTIONS file
|
||||
if 'transaction' in file_name:
|
||||
print(f'Processing "transaction" file: {f}')
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
line_count = 0
|
||||
for row in csv_reader:
|
||||
if line_count == 0:
|
||||
line_count += 1
|
||||
else:
|
||||
try:
|
||||
print(f'Processing line {line_count}: {row}')
|
||||
user_id, card, phone, summ_total, summ_discount, sum_with_discount, bonus_amount, transaction_date, transaction_time = row
|
||||
# TRANSACTIONS files
|
||||
start_time = time.time()
|
||||
if 'transaction' in file_name:
|
||||
print(f'Processing "transaction" file: {f}')
|
||||
csv_reader = csv.reader(csv_file, delimiter=',')
|
||||
line_count = 0
|
||||
for row in csv_reader:
|
||||
if line_count == 0:
|
||||
line_count += 1
|
||||
else:
|
||||
try:
|
||||
line_count += 1
|
||||
print(f'Processing line {line_count}: {row}')
|
||||
user_id, card, phone, summ_total, summ_discount, sum_with_discount, bonus_amount, transaction_date, transaction_time, doc_id = row
|
||||
|
||||
except ValueError as e:
|
||||
ret = f'error in line: [{line_count}] {repr(e)}'
|
||||
log_file.write(f'{ret}\n')
|
||||
if card.strip() != '':
|
||||
user_found_by_card, din_id, _, _, _ = get_user(card, get_type='auto')
|
||||
if not user_found_by_card:
|
||||
print(f'error in line: [{line_count}]- Invalid card: {card}')
|
||||
log_file.write(f'error in line: [{line_count}]- Invalid card: {card}\n')
|
||||
else:
|
||||
print('User found with', user_id, card)
|
||||
else:
|
||||
user_found_by_card = False
|
||||
|
||||
csv_file.close()
|
||||
log_file.close()
|
||||
if validate_phone(phone) != '':
|
||||
phone = phone.replace('+', '')
|
||||
user_found_by_phone, din_id, _, _, _ = get_user(phone, get_type='phone')
|
||||
|
||||
# os.rename(f, f + '.old')
|
||||
if not user_found_by_phone:
|
||||
print(f'error in line: [{line_count}]- Invalid phone: {phone}')
|
||||
log_file.write(f'error in line: [{line_count}]- Invalid phone: {phone}\n')
|
||||
else:
|
||||
print('User found with', din_id, phone)
|
||||
else:
|
||||
user_found_by_phone = False
|
||||
|
||||
user_found = user_found_by_card or user_found_by_phone
|
||||
if user_found:
|
||||
if doc_id.strip() == '':
|
||||
doc_id = f'{file_name[0:20]}-{line_count}-{card}'
|
||||
else:
|
||||
doc_id = doc_id.strip()
|
||||
|
||||
result, data = bonuses_update(
|
||||
user_id=din_id,
|
||||
summ_total=summ_total,
|
||||
bonus_amount=bonus_amount,
|
||||
doc_id=doc_id,
|
||||
sum_with_discount=sum_with_discount,
|
||||
sum_discount=summ_discount,
|
||||
dry_run=DRY_RUN
|
||||
)
|
||||
|
||||
if not result:
|
||||
print(f'error in line: [{line_count}]- bonuses_update: {data}')
|
||||
log_file.write(f'error in line: [{line_count}]- bonuses_update: {data}\n')
|
||||
else:
|
||||
print(f'Bonuses updated, user_id={din_id}')
|
||||
|
||||
else:
|
||||
print(f'error in line: [{line_count}]- Invalid user: {user_id}')
|
||||
log_file.write(f'error in line: [{line_count}]- Invalid user: {user_id}\n')
|
||||
|
||||
except ValueError as e:
|
||||
ret = f'error in line: [{line_count}] {repr(e)}'
|
||||
log_file.write(f'{ret}\n')
|
||||
|
||||
end_time = time.time()
|
||||
print(f'Elapsed time of TRANSACTIONS file processing : {end_time - start_time} seconds')
|
||||
|
||||
csv_file.close()
|
||||
log_file.close()
|
||||
|
||||
os.rename(f, f + '.' + time.strftime("%Y%m%d-%H%M%S") + '.old')
|
||||
|
||||
|
||||
scheduler = BackgroundScheduler()
|
||||
scheduler.start()
|
||||
|
||||
try:
|
||||
job = scheduler.add_job(run_import, 'interval', minutes=1)
|
||||
print('Import running every 1 minute. Place *.csv files in "csv" directory')
|
||||
print('Press Ctrl+{0} to exit'.format('Break' if apscheduler.__version__ >= '3.0.0' else 'C'))
|
||||
# This is here to simulate application activity (which keeps the main thread alive).
|
||||
while True:
|
||||
time.sleep(10)
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
# Not strictly necessary if daemonic mode is enabled but should be done if possible
|
||||
scheduler.shutdown()
|
||||
|
||||
Reference in New Issue
Block a user