xgiovio/openstack-swift-backup-restore-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed comparison and download on encrypted files. delete 0kb files after os.rename (may be a bug)

Browse Source
This commit is contained in:
Giovanni Di Grezia
2015-12-03 10:29:01 +01:00
parent 0fee56afd0
commit a68470dcac
3 changed files with 27 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
local_to_swift.py
View File

@@ -4,6 +4,7 @@ import authentication
import utility import utility
import os, sys,platform, math, hashlib,time import os, sys,platform, math, hashlib,time
from utility import read_in_chunks from utility import read_in_chunks
from time import time
import shutil import shutil
def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir): def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir):
@@ -70,13 +71,13 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
for lname in localfiles.keys(): for lname in localfiles.keys():
#print("Comparing " + lname) #print("Comparing " + lname)
upload_file = True upload_file = True
#80 is the maximum string length appended to an encrypted file name : "_xg10v10_fd3b094fd9c48c6ee288f58c991dec9d_xg10v10_999000000000_xg10v10_encrypted"
if not uploadlarge and localfiles[lname] > size_limit_to_segment: if not uploadlarge and localfiles[lname] > size_limit_to_segment:
upload_file = False upload_file = False
print("Skipped " + lname + " Upload of large files is disabled. Limit " + size_limit_to_segment) print("Skipped " + lname + " Upload of large files is disabled. Limit " + size_limit_to_segment)
skipped_large_files = skipped_large_files + 1 skipped_large_files = skipped_large_files + 1
elif encrypted and (len(utility.file_only_name(lname,utility.set_dash())) + 80 ) > 255 : #105 is the maximum string length appended to an encrypted file name : "_xg10v10_fd3b094fd9c48c6ee288f58c991dec9d_xg10v10_999000000000_xg10v10_1449108838_xg10v10_encrypted_0001"
print("Skipped " + lname + " encryption: The name is too long for the filesystem " + str(len(utility.file_only_name(lname,utility.set_dash())) + 80)) elif encrypted and (len(utility.file_only_name(lname,utility.set_dash())) + 105 ) > 255 :
print("Skipped " + lname + " encryption: The name is too long for the filesystem " + str(len(utility.file_only_name(lname,utility.set_dash())) + 105))
encrypted_skipped_error = encrypted_skipped_error + 1 encrypted_skipped_error = encrypted_skipped_error + 1
upload_file = False upload_file = False
else: else:
@@ -89,6 +90,7 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
if upload_file: if upload_file:
if encrypted : if encrypted :
lnameenc = lname + "_xg10v10_encrypted" lnameenc = lname + "_xg10v10_encrypted"
xtime = str(int(time()))
if lnameenc not in remotefiles_encr.keys() or localfiles[lname] != int((remotefiles_encr[lnameenc]).split("_xg10v10_")[2]) or remotefiles[remotefiles_encr[lnameenc]] != utility.total_size_encrypted(localfiles[lname]) : if lnameenc not in remotefiles_encr.keys() or localfiles[lname] != int((remotefiles_encr[lnameenc]).split("_xg10v10_")[2]) or remotefiles[remotefiles_encr[lnameenc]] != utility.total_size_encrypted(localfiles[lname]) :
if upload: if upload:
with open(localpath + lname, 'rb') as f: with open(localpath + lname, 'rb') as f:
@@ -97,12 +99,15 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
with open(temp_dir + lname, 'wb') as d: with open(temp_dir + lname, 'wb') as d:
aes = utility_aes.AESCipherFile(f,d, size_limit_reading_os, encrypt_key) aes = utility_aes.AESCipherFile(f,d, size_limit_reading_os, encrypt_key)
md5 = aes.encrypt() md5 = aes.encrypt()
if os.path.isfile(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"): if os.path.isfile(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"):
os.remove(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted") os.remove(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted")
os.rename (temp_dir + lname, temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted") os.rename (temp_dir + lname, temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted")
difffiles[lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname]) #double check os.rename has removed the old file. maybe a bug
if os.path.isfile(temp_dir + lname):
os.remove(temp_dir + lname)
difffiles[lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname])
else : else :
difffiles[lname + "_xg10v10_" + "md5_not_calculated" + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname]) difffiles[lname + "_xg10v10_" + "md5_not_calculated" + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname])
sizetoupload = sizetoupload + utility.total_size_encrypted(localfiles[lname]) sizetoupload = sizetoupload + utility.total_size_encrypted(localfiles[lname])
filestoupload = filestoupload + 1 filestoupload = filestoupload + 1
if (localfiles[lname] > size_limit_to_segment): if (localfiles[lname] > size_limit_to_segment):
@@ -119,10 +124,13 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
with open(localpath + lname, 'rb') as f: with open(localpath + lname, 'rb') as f:
with open(temp_dir + lname, 'wb') as d: with open(temp_dir + lname, 'wb') as d:
aes = utility_aes.AESCipherFile(f,d, size_limit_reading_os, encrypt_key) aes = utility_aes.AESCipherFile(f,d, size_limit_reading_os, encrypt_key)
if os.path.isfile(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"): if os.path.isfile(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"):
os.remove(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted") os.remove(temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted")
os.rename (temp_dir + lname, temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted") os.rename (temp_dir + lname, temp_dir + lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted")
difffiles[lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname]) #double check os.rename has removed the old file. maybe a bug
if os.path.isfile(temp_dir + lname):
os.remove(temp_dir + lname)
difffiles[lname + "_xg10v10_" + md5 + "_xg10v10_" + str(localfiles[lname]) + "_xg10v10_" + xtime + "_xg10v10_encrypted"] = utility.total_size_encrypted(localfiles[lname])
sizetoupload = sizetoupload + utility.total_size_encrypted(localfiles[lname]) sizetoupload = sizetoupload + utility.total_size_encrypted(localfiles[lname])
filestoupload = filestoupload + 1 filestoupload = filestoupload + 1
if (localfiles[lname] > size_limit_to_segment): if (localfiles[lname] > size_limit_to_segment):

3
swift_to_local.py
View File

@@ -35,6 +35,7 @@ def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_
swift_conn = authentication.set_authentication () swift_conn = authentication.set_authentication ()
swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix) swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix)
byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5 = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix) byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5 = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix)
remotefiles_encr = utility.list_compute_correct_names_for_enctyption(objects,prefix)
print ("Files remoti " + str(len(remotefiles))) print ("Files remoti " + str(len(remotefiles)))
@@ -45,7 +46,7 @@ def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_
skipped_old_md5_large_files = 0 skipped_old_md5_large_files = 0
if encrypted and download : if encrypted and download :
import utility_aes import utility_aes
for rname in remotefiles.keys(): for rname in remotefiles_encr.values():
download_file = True download_file = True
for pattern in excluded_patterns: for pattern in excluded_patterns:
if pattern in rname: if pattern in rname:

7
utility.py
View File

@@ -113,8 +113,11 @@ def list_compute_correct_names_for_enctyption (objects,prefix):
encrypted_name_only = full.split("_xg10v10_")[0] + "_xg10v10_encrypted" encrypted_name_only = full.split("_xg10v10_")[0] + "_xg10v10_encrypted"
else: else:
encrypted_name_only = full encrypted_name_only = full
remotefiles[encrypted_name_only] = full if encrypted_name_only in remotefiles.keys():
if int(remotefiles[encrypted_name_only].split("_xg10v10_")[3]) < int(full.split("_xg10v10_")[3]) :
remotefiles[encrypted_name_only] = full
else:
remotefiles[encrypted_name_only] = full
return remotefiles return remotefiles