162 lines
7.8 KiB
Python
162 lines
7.8 KiB
Python
__author__ = 'xgiovio'
|
|
|
|
import authentication
|
|
import utility
|
|
import os, sys,platform, hashlib,time
|
|
from utility import read_in_chunks
|
|
|
|
def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns):
|
|
print ("Localpath " + localpath)
|
|
print ("Swift container " + swift_container)
|
|
print ("Swift prefix " + prefix)
|
|
print ("Os reading Limit " + str(size_limit_reading_os))
|
|
print ("Fail tries " + str(fail_tries))
|
|
print ("Download " + str(download))
|
|
print ("MD5 Compare " + str(md5_compare))
|
|
print ("Encrypted " + str(encrypted))
|
|
if encrypted:
|
|
if encrypt_key == None:
|
|
print("Error Bad Encryption Key")
|
|
sys.exit(-3)
|
|
else:
|
|
print ("Encrypted key " + "hidden")
|
|
print("___________")
|
|
|
|
|
|
|
|
#checks
|
|
utility.check_segments_size_single (size_limit_reading_os)
|
|
if utility.check_start_slash(swift_container) or utility.check_start_slash(prefix) or not utility.check_end_slash(localpath) or utility.check_end_slash(swift_container) or not utility.check_end_slash(prefix):
|
|
print ("Errors on local or remote paths. Checks \\ or / at the begin or end of each path")
|
|
sys.exit(-2)
|
|
localfiles = utility.get_local_files (localpath)
|
|
print ("Files locali " + str(len(localfiles)))
|
|
print("___________")
|
|
swift_conn = authentication.set_authentication ()
|
|
swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix)
|
|
byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5 = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix)
|
|
remotefiles_encr = utility.list_compute_correct_names_for_enctyption(objects,prefix)
|
|
|
|
print ("Files remoti " + str(len(remotefiles)))
|
|
|
|
difffiles = {}
|
|
skipped = 0
|
|
sizetodownload=0
|
|
filestodownload = 0
|
|
skipped_old_md5_large_files = 0
|
|
if encrypted and download :
|
|
import utility_aes
|
|
for rname in remotefiles_encr.values():
|
|
download_file = True
|
|
for pattern in excluded_patterns:
|
|
if pattern in rname:
|
|
download_file = False
|
|
break
|
|
if download_file:
|
|
#print("Comparing " + lname)
|
|
if encrypted and rname.endswith("_xg10v10_encrypted"):
|
|
rnamedec = rname.split("_xg10v10_")[0]
|
|
rnamesize = rname.split("_xg10v10_")[2]
|
|
rnamemd5 = rname.split("_xg10v10_")[1]
|
|
if rnamedec not in localfiles.keys() or localfiles[rnamedec] != int(rnamesize) :
|
|
difffiles[rname] = remotefiles[rname]
|
|
sizetodownload = sizetodownload + remotefiles[rname]
|
|
filestodownload = filestodownload + 1
|
|
elif md5_compare :
|
|
hash = hashlib.md5()
|
|
with open(localpath + rnamedec, 'rb') as f:
|
|
for piece in read_in_chunks(f,size_limit_reading_os):
|
|
hash.update(piece)
|
|
if hash.hexdigest() != rnamemd5:
|
|
difffiles[rname] = remotefiles[rname]
|
|
sizetodownload = sizetodownload + remotefiles[rname]
|
|
filestodownload = filestodownload + 1
|
|
else:
|
|
if rname not in localfiles.keys() or localfiles[rname] != remotefiles[rname] :
|
|
difffiles[rname] = remotefiles[rname]
|
|
sizetodownload = sizetodownload + remotefiles[rname]
|
|
filestodownload = filestodownload + 1
|
|
elif md5_compare :
|
|
if remotefiles_md5[rname] != "0":
|
|
hash = hashlib.md5()
|
|
with open(localpath + rname, 'rb') as f:
|
|
for piece in read_in_chunks(f,size_limit_reading_os):
|
|
hash.update(piece)
|
|
if hash.hexdigest() != remotefiles_md5[rname] :
|
|
difffiles[rname] = remotefiles[rname]
|
|
sizetodownload = sizetodownload + remotefiles[rname]
|
|
filestodownload = filestodownload + 1
|
|
else:
|
|
skipped_old_md5_large_files = skipped_old_md5_large_files + 1
|
|
else:
|
|
print("Skipped " + rname + " due to filters" )
|
|
skipped = skipped + 1
|
|
|
|
print("___________Differenze___________")
|
|
print ("Files locali " + str(len(localfiles)))
|
|
print ("Skipped due to filters " + str(skipped))
|
|
if md5_compare:
|
|
print ("Skipped md5 comparison on old large files without md5 embedded in x-object-manifest: " + str(skipped_old_md5_large_files))
|
|
print ("Files remoti " + str(len(remotefiles)))
|
|
print ("Files remoti - skipped " + str(len(remotefiles) - skipped))
|
|
print ("Files remoti 0byte reali " + str(byte0real))
|
|
print ("Files remoti 0byte large " + str(byte0manifest))
|
|
print("___________Files to download______")
|
|
for files, sizes in difffiles.items():
|
|
if encrypted and files.endswith("_xg10v10_encrypted"):
|
|
print(files.split("_xg10v10_")[0],sizes)
|
|
else:
|
|
print(files,sizes)
|
|
print("___________")
|
|
print ("MBytes to download " + str(sizetodownload/1000000))
|
|
print ("Files to download " + str(filestodownload))
|
|
print("___________")
|
|
|
|
remainingtodownload = sizetodownload
|
|
if download :
|
|
errors_download = 0
|
|
skipped_downloads = 0
|
|
for file, size in difffiles.items():
|
|
for fail_tries_counter in range (fail_tries) :
|
|
try:
|
|
print("Downloading File: " + utility.dash_replace(prefix + file) + " " + str(size) )
|
|
f = swift_conn.get_object(swift_container,utility.dash_replace(prefix + file),resp_chunk_size=size_limit_reading_os)
|
|
if not os.path.isdir(utility.folder_from_path(localpath + file,utility.set_dash())[:-1]):
|
|
os.makedirs(utility.folder_from_path(localpath + file,utility.set_dash())[:-1])
|
|
with open( localpath + file,'wb') as t:
|
|
while True:
|
|
try:
|
|
t.write(f[1].next())
|
|
except StopIteration:
|
|
break
|
|
if encrypted and file.endswith("_xg10v10_encrypted"):
|
|
rnamedec = file.split("_xg10v10_")[0]
|
|
with open(localpath + file, 'rb') as f:
|
|
with open(localpath + rnamedec, 'wb') as d:
|
|
aes = utility_aes.AESCipherFile(f,d, size_limit_reading_os, encrypt_key)
|
|
aes.decrypt()
|
|
os.remove(localpath + file)
|
|
print("Decrypted")
|
|
remainingtodownload = remainingtodownload - size
|
|
print("Remaining to download : " + str(remainingtodownload / 1000000) + " MB")
|
|
except Exception as e:
|
|
print("Exception during download")
|
|
print(e)
|
|
time.sleep(1)
|
|
errors_download = errors_download + 1
|
|
swift_conn = authentication.set_authentication ()
|
|
if fail_tries_counter == fail_tries - 1 :
|
|
print("Maximum tries reached. Skipping download of the file")
|
|
skipped_downloads = skipped_downloads + 1
|
|
else:
|
|
break
|
|
print("___________")
|
|
print("Download Terminated : Remaining MB to download " + str(remainingtodownload))
|
|
print("Errors during download : " + str(errors_download))
|
|
print("Skipped files during download : " + str(skipped_downloads))
|
|
|
|
else:
|
|
print("Download Disabled")
|
|
|
|
swift_conn.close()
|