xgiovio/openstack-swift-backup-restore-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

return false instead of exiting in some functions. Added modification to split for md5

Browse Source
This commit is contained in:
Giovanni Di Grezia
2018-05-15 21:16:56 +02:00
parent 40753a8b96
commit 438a248c49
5 changed files with 43 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
local_to_swift.py
View File

@@ -38,7 +38,10 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
#checks
utility.check_segments_size (size_limit_reading_os,size_limit_to_segment)
if not utility.check_segments_size (size_limit_reading_os,size_limit_to_segment)
print ("Error size segments")
sys.exit(-2)
if utility.check_start_slash(swift_container) or utility.check_start_slash(prefix) or not utility.check_end_slash(localpath) or not utility.check_end_slash(temp_dir) or utility.check_end_slash(swift_container) or not utility.check_end_slash(prefix):
print ("Errors on local or remote paths. Checks \\ or / at the begin or end of each path")
sys.exit(-2)
@@ -46,8 +49,14 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
print ("Files locali " + str(len(localfiles)))
print("___________")
swift_conn = authentication.set_authentication ()
swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix)
byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix)
results = utility.get_list(fail_tries,swift_conn,swift_container,prefix)
if not results:
return
swift_conn,objects = results
results = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix)
if not results:
return
byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj = results
if encrypted:
remotefiles_encr,list_enc_old = utility.list_compute_correct_names_for_encryption(objects,prefix)
@@ -326,7 +335,10 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
print("Computing deletion list...")
if encrypted:
#update remote list with new files encrypted
swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix)
results = utility.get_list(fail_tries,swift_conn,swift_container,prefix)
if not results:
return
swift_conn,objects = results
remotefiles_encr,list_enc_old = utility.list_compute_correct_names_for_encryption(objects,prefix)
for o in list_enc_old:

5
swift_delete_0_byte_folders.py
View File

@@ -8,7 +8,10 @@ def launch(swift_container,delete,fail_tries):
print ("Fail tries " + str(fail_tries))
print("___________")
swift_conn = authentication.set_authentication()
swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,"")
results = utility.get_list(fail_tries,swift_conn,swift_container,"")
if not results:
return
swift_conn,objects = results
remotefiles = utility.list_compute_0_byte_folders(objects)

10
swift_delete_orphan_segments.py
View File

@@ -14,7 +14,10 @@ def launch(swift_container,delete,fail_tries):
listfoldermanifest=[]
#all files on the container
swift_conn,container_objects_raw = utility.get_list(fail_tries,swift_conn,swift_container,"")
results = utility.get_list(fail_tries,swift_conn,swift_container,"")
if not results:
return
swift_conn,container_objects_raw = results
#get dict name:manifesturl only for large files
byte0manifest,swift_conn,container_objects_manifest = utility.list_compute_manifest (fail_tries,container_objects_raw,swift_conn,swift_container,"")
for name,manifest in container_objects_manifest.items():
@@ -31,7 +34,10 @@ def launch(swift_container,delete,fail_tries):
#______________________________Folders with segments container: swift_container +"_segments"
#all files on container_segments -> they are only segments
swift_conn,container_segments_objects_raw = utility.get_list(fail_tries,swift_conn,swift_container +"_segments","")
results = utility.get_list(fail_tries,swift_conn,swift_container +"_segments","")
if not results :
return
swift_conn,container_segments_objects_raw = results
#get dict name:size
container_segments_objects = utility.list (container_segments_objects_raw,"")
#get list only with folders path name from segments

10
swift_to_local.py
View File

@@ -33,8 +33,14 @@ def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_
print ("Files locali " + str(len(localfiles)))
print("___________")
swift_conn = authentication.set_authentication ()
swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix)
byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix)
results = utility.get_list(fail_tries,swift_conn,swift_container,prefix)
if not results:
return
swift_conn,objects = results
results = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix)
if not results:
return
byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj = results
remotefiles_encr,list_enc_old = utility.list_compute_correct_names_for_encryption(objects,prefix)
print ("Files remoti " + str(len(remotefiles)))

14
utility.py
View File

@@ -31,8 +31,8 @@ def get_list (fail_tries,swift_conn,swift_container,prefix) :
print(e)
time.sleep(2)
if fail_tries_counter == fail_tries - 1 :
print("Maximum tries reached. Can't download remote list for container " + swift_container + " with prefix " +prefix +". Exiting.")
sys.exit(-1)
print("Maximum tries reached. Can't download remote list for container " + swift_container + " with prefix " +prefix +".")
return False
else:
swift_conn = authentication.set_authentication ()
else :
@@ -76,8 +76,8 @@ def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,pre
print(e)
time.sleep(2)
if fail_tries_counter == fail_tries - 1 :
print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.")
sys.exit(-1)
print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +".")
return False
else:
swift_conn = authentication.set_authentication ()
else :
@@ -87,7 +87,7 @@ def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,pre
o["bytes"] = oheaders["content-length"]
if "_xg10v10_" in oheaders["x-object-manifest"]:
split1 = oheaders["x-object-manifest"].split("_xg10v10_")[0]
split2 = split1.split("/")[1]
split2 = split1.split("/")[-1]
o["hash"] = split2
else:
print("Impossible to get remote large file md5. Cause: Not uploaded with xgiovio method (md5 in x-object-manifest)")
@@ -177,8 +177,8 @@ def listall (objects,prefix):
def check_segments_size (size_limit_reading_os,size_limit_to_segment):
if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16 or not is_power_of_2(size_limit_reading_os) or not is_power_of_2(size_limit_to_segment):
print ("Error size segments")
sys.exit(-2)
return False
return True
def is_power_of_2 (number):
result = math.log(number,2)