diff --git a/local_to_swift.py b/local_to_swift.py index b137d82..f0acb01 100644 --- a/local_to_swift.py +++ b/local_to_swift.py @@ -4,6 +4,7 @@ import authentication import utility import os, sys,platform, math, hashlib,time from utility import read_in_chunks +from time import time import shutil def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir): @@ -70,13 +71,13 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ for lname in localfiles.keys(): #print("Comparing " + lname) upload_file = True - #80 is the maximum string length appended to an encrypted file name : "_xg10v10_fd3b094fd9c48c6ee288f58c991dec9d_xg10v10_999000000000_xg10v10_encrypted" if not uploadlarge and localfiles[lname] > size_limit_to_segment: upload_file = False print("Skipped " + lname + " Upload of large files is disabled. Limit " + size_limit_to_segment) skipped_large_files = skipped_large_files + 1 - elif encrypted and (len(utility.file_only_name(lname,utility.set_dash())) + 80 ) > 255 : - print("Skipped " + lname + " encryption: The name is too long for the filesystem " + str(len(utility.file_only_name(lname,utility.set_dash())) + 80)) + #105 is the maximum string length appended to an encrypted file name : "_xg10v10_fd3b094fd9c48c6ee288f58c991dec9d_xg10v10_999000000000_xg10v10_1449108838_xg10v10_encrypted_0001" + elif encrypted and (len(utility.file_only_name(lname,utility.set_dash())) + 105 ) > 255 : + print("Skipped " + lname + " encryption: The name is too long for the filesystem " + str(len(utility.file_only_name(lname,utility.set_dash())) + 105)) encrypted_skipped_error = encrypted_skipped_error + 1 upload_file = False else: @@ -89,6 +90,7 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ if upload_file: if encrypted : lnameenc = lname + "_xg10v10_encrypted" + xtime = str(int(time())) if lnameenc not in remotefiles_encr.keys() or localfiles[lname] != int((remotefiles_encr[lnameenc]).split("_xg10v10_")[2]) or remotefiles[remotefiles_encr[lnameenc]] != utility.total_size_encrypted(localfiles[lname]) : if upload: with open(localpath + lname, 'rb') as f: @@ -97,12 +99,15 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ with open(temp_dir + lname, 'wb') as d: aes = utility_aes.AESCipherFile(f,d, size_limit_reading_os, encrypt_key) md5 = aes.encrypt() - if os.path.isfile(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"): - os.remove(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted") - os.rename (temp_dir + lname, temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted") - difffiles[lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname]) + if os.path.isfile(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"): + os.remove(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted") + os.rename (temp_dir + lname, temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted") + #double check os.rename has removed the old file. maybe a bug + if os.path.isfile(temp_dir + lname): + os.remove(temp_dir + lname) + difffiles[lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname]) else : - difffiles[lname + "_xg10v10_" + "md5_not_calculated" + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname]) + difffiles[lname + "_xg10v10_" + "md5_not_calculated" + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname]) sizetoupload = sizetoupload + utility.total_size_encrypted(localfiles[lname]) filestoupload = filestoupload + 1 if (localfiles[lname] > size_limit_to_segment): @@ -119,10 +124,13 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ with open(localpath + lname, 'rb') as f: with open(temp_dir + lname, 'wb') as d: aes = utility_aes.AESCipherFile(f,d, size_limit_reading_os, encrypt_key) - if os.path.isfile(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"): - os.remove(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted") - os.rename (temp_dir + lname, temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted") - difffiles[lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname]) + if os.path.isfile(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"): + os.remove(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted") + os.rename (temp_dir + lname, temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted") + #double check os.rename has removed the old file. maybe a bug + if os.path.isfile(temp_dir + lname): + os.remove(temp_dir + lname) + difffiles[lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname]) sizetoupload = sizetoupload + utility.total_size_encrypted(localfiles[lname]) filestoupload = filestoupload + 1 if (localfiles[lname] > size_limit_to_segment): diff --git a/swift_to_local.py b/swift_to_local.py index c7f049c..39cdeb1 100644 --- a/swift_to_local.py +++ b/swift_to_local.py @@ -35,6 +35,7 @@ def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_ swift_conn = authentication.set_authentication () swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix) byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5 = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix) + remotefiles_encr = utility.list_compute_correct_names_for_enctyption(objects,prefix) print ("Files remoti " + str(len(remotefiles))) @@ -45,7 +46,7 @@ def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_ skipped_old_md5_large_files = 0 if encrypted and download : import utility_aes - for rname in remotefiles.keys(): + for rname in remotefiles_encr.values(): download_file = True for pattern in excluded_patterns: if pattern in rname: diff --git a/utility.py b/utility.py index 9450497..7a2ab76 100644 --- a/utility.py +++ b/utility.py @@ -113,8 +113,11 @@ def list_compute_correct_names_for_enctyption (objects,prefix): encrypted_name_only = full.split("_xg10v10_")[0] + "_xg10v10_encrypted" else: encrypted_name_only = full - remotefiles[encrypted_name_only] = full - + if encrypted_name_only in remotefiles.keys(): + if int(remotefiles[encrypted_name_only].split("_xg10v10_")[3]) < int(full.split("_xg10v10_")[3]) : + remotefiles[encrypted_name_only] = full + else: + remotefiles[encrypted_name_only] = full return remotefiles