first release. readme will follow up
This commit is contained in:
160
swift_to_local.py
Normal file
160
swift_to_local.py
Normal file
@@ -0,0 +1,160 @@
|
||||
__author__ = 'xgiovio'
|
||||
|
||||
import authentication
|
||||
import utility
|
||||
import os, sys,platform, hashlib,time
|
||||
from utility import read_in_chunks
|
||||
|
||||
def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns):
|
||||
print ("Localpath " + localpath)
|
||||
print ("Swift container " + swift_container)
|
||||
print ("Swift prefix " + prefix)
|
||||
print ("Os reading Limit " + str(size_limit_reading_os))
|
||||
print ("Fail tries " + str(fail_tries))
|
||||
print ("Download " + str(download))
|
||||
print ("MD5 Compare " + str(md5_compare))
|
||||
print ("Encrypted " + str(encrypted))
|
||||
if encrypted:
|
||||
if encrypt_key == None:
|
||||
print("Error Bad Encryption Key")
|
||||
sys.exit(-3)
|
||||
else:
|
||||
print ("Encrypted key " + "hidden")
|
||||
print("___________")
|
||||
|
||||
|
||||
|
||||
#checks
|
||||
utility.check_segments_size_single (size_limit_reading_os)
|
||||
if utility.check_start_slash(swift_container) or utility.check_start_slash(prefix) or not utility.check_end_slash(localpath) or utility.check_end_slash(swift_container) or not utility.check_end_slash(prefix):
|
||||
print ("Errors on local or remote paths. Checks \\ or / at the begin or end of each path")
|
||||
sys.exit(-2)
|
||||
localfiles = utility.get_local_files (localpath)
|
||||
print ("Files locali " + str(len(localfiles)))
|
||||
print("___________")
|
||||
swift_conn = authentication.set_authentication ()
|
||||
swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix)
|
||||
byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5 = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix)
|
||||
|
||||
print ("Files remoti " + str(len(remotefiles)))
|
||||
|
||||
difffiles = {}
|
||||
skipped = 0
|
||||
sizetodownload=0
|
||||
filestodownload = 0
|
||||
skipped_old_md5_large_files = 0
|
||||
if encrypted and download :
|
||||
import utility_aes
|
||||
for rname in remotefiles.keys():
|
||||
download_file = True
|
||||
for pattern in excluded_patterns:
|
||||
if pattern in rname:
|
||||
download_file = False
|
||||
break
|
||||
if download_file:
|
||||
#print("Comparing " + lname)
|
||||
if encrypted and rname.endswith("_xg10v10_encrypted"):
|
||||
rnamedec = rname.split("_xg10v10_")[0]
|
||||
rnamesize = rname.split("_xg10v10_")[2]
|
||||
rnamemd5 = rname.split("_xg10v10_")[1]
|
||||
if rnamedec not in localfiles.keys() or localfiles[rnamedec] != int(rnamesize) :
|
||||
difffiles[rname] = remotefiles[rname]
|
||||
sizetodownload = sizetodownload + remotefiles[rname]
|
||||
filestodownload = filestodownload + 1
|
||||
elif md5_compare :
|
||||
hash = hashlib.md5()
|
||||
with open(localpath + rnamedec, 'rb') as f:
|
||||
for piece in read_in_chunks(f,size_limit_reading_os):
|
||||
hash.update(piece)
|
||||
if hash.hexdigest() != rnamemd5:
|
||||
difffiles[rname] = remotefiles[rname]
|
||||
sizetodownload = sizetodownload + remotefiles[rname]
|
||||
filestodownload = filestodownload + 1
|
||||
else:
|
||||
if rname not in localfiles.keys() or localfiles[rname] != remotefiles[rname] :
|
||||
difffiles[rname] = remotefiles[rname]
|
||||
sizetodownload = sizetodownload + remotefiles[rname]
|
||||
filestodownload = filestodownload + 1
|
||||
elif md5_compare :
|
||||
if remotefiles_md5[rname] != "0":
|
||||
hash = hashlib.md5()
|
||||
with open(localpath + rname, 'rb') as f:
|
||||
for piece in read_in_chunks(f,size_limit_reading_os):
|
||||
hash.update(piece)
|
||||
if hash.hexdigest() != remotefiles_md5[rname] :
|
||||
difffiles[rname] = remotefiles[rname]
|
||||
sizetodownload = sizetodownload + remotefiles[rname]
|
||||
filestodownload = filestodownload + 1
|
||||
else:
|
||||
skipped_old_md5_large_files = skipped_old_md5_large_files + 1
|
||||
else:
|
||||
print("Skipped " + rname + " due to filters" )
|
||||
skipped = skipped + 1
|
||||
|
||||
print("___________Differenze___________")
|
||||
print ("Files locali " + str(len(localfiles)))
|
||||
print ("Skipped due to filters " + str(skipped))
|
||||
if md5_compare:
|
||||
print ("Skipped md5 comparison on old large files without md5 embedded in x-object-manifest: " + str(skipped_old_md5_large_files))
|
||||
print ("Files remoti " + str(len(remotefiles)))
|
||||
print ("Files remoti - skipped " + str(len(remotefiles) - skipped))
|
||||
print ("Files remoti 0byte reali " + str(byte0real))
|
||||
print ("Files remoti 0byte large " + str(byte0manifest))
|
||||
print("___________Files to download______")
|
||||
for files, sizes in difffiles.items():
|
||||
if encrypted and files.endswith("_xg10v10_encrypted"):
|
||||
print(files.split("_xg10v10_")[0],sizes)
|
||||
else:
|
||||
print(files,sizes)
|
||||
print("___________")
|
||||
print ("MBytes to download " + str(sizetodownload/1000000))
|
||||
print ("Files to download " + str(filestodownload))
|
||||
print("___________")
|
||||
|
||||
remainingtodownload = sizetodownload
|
||||
if download :
|
||||
errors_download = 0
|
||||
skipped_downloads = 0
|
||||
for file, size in difffiles.items():
|
||||
for fail_tries_counter in range (fail_tries) :
|
||||
try:
|
||||
print("Downloading File: " + utility.dash_replace(prefix + file) + " " + str(size) )
|
||||
f = swift_conn.get_object(swift_container,utility.dash_replace(prefix + file),resp_chunk_size=size_limit_reading_os)
|
||||
if not os.path.isdir(utility.folder_from_path(localpath + file,utility.set_dash())[:-1]):
|
||||
os.makedirs(utility.folder_from_path(localpath + file,utility.set_dash())[:-1])
|
||||
with open( localpath + file,'wb') as t:
|
||||
while True:
|
||||
try:
|
||||
t.write(f[1].next())
|
||||
except StopIteration:
|
||||
break
|
||||
if encrypted and file.endswith("_xg10v10_encrypted"):
|
||||
rnamedec = file.split("_xg10v10_")[0]
|
||||
with open(localpath + file, 'rb') as f:
|
||||
with open(localpath + rnamedec, 'wb') as d:
|
||||
aes = utility_aes.AESCipherFile(f,d, size_limit_reading_os, encrypt_key)
|
||||
aes.decrypt()
|
||||
os.remove(localpath + file)
|
||||
print("Decrypted")
|
||||
remainingtodownload = remainingtodownload - size
|
||||
print("Remaining to download : " + str(remainingtodownload / 1000000) + " MB")
|
||||
except Exception as e:
|
||||
print("Exception during download")
|
||||
print(e)
|
||||
time.sleep(1)
|
||||
errors_download = errors_download + 1
|
||||
swift_conn = authentication.set_authentication ()
|
||||
if fail_tries_counter == fail_tries - 1 :
|
||||
print("Maximum tries reached. Skipping download of the file")
|
||||
skipped_downloads = skipped_downloads + 1
|
||||
else:
|
||||
break
|
||||
print("___________")
|
||||
print("Download Terminated : Remaining MB to download " + str(remainingtodownload))
|
||||
print("Errors during download : " + str(errors_download))
|
||||
print("Skipped files during download : " + str(skipped_downloads))
|
||||
|
||||
else:
|
||||
print("Download Disabled")
|
||||
|
||||
swift_conn.close()
|
||||
Reference in New Issue
Block a user