From 438a248c49457bd6a3cdf8de52afd36e661d27a2 Mon Sep 17 00:00:00 2001 From: Giovanni Di Grezia Date: Tue, 15 May 2018 21:16:56 +0200 Subject: [PATCH] return false instead of exiting in some functions. Added modification to split for md5 --- local_to_swift.py | 20 ++++++++++++++++---- swift_delete_0_byte_folders.py | 5 ++++- swift_delete_orphan_segments.py | 10 ++++++++-- swift_to_local.py | 10 ++++++++-- utility.py | 14 +++++++------- 5 files changed, 43 insertions(+), 16 deletions(-) diff --git a/local_to_swift.py b/local_to_swift.py index f37f0a1..76af696 100644 --- a/local_to_swift.py +++ b/local_to_swift.py @@ -38,7 +38,10 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ #checks - utility.check_segments_size (size_limit_reading_os,size_limit_to_segment) + + if not utility.check_segments_size (size_limit_reading_os,size_limit_to_segment) + print ("Error size segments") + sys.exit(-2) if utility.check_start_slash(swift_container) or utility.check_start_slash(prefix) or not utility.check_end_slash(localpath) or not utility.check_end_slash(temp_dir) or utility.check_end_slash(swift_container) or not utility.check_end_slash(prefix): print ("Errors on local or remote paths. Checks \\ or / at the begin or end of each path") sys.exit(-2) @@ -46,8 +49,14 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ print ("Files locali " + str(len(localfiles))) print("___________") swift_conn = authentication.set_authentication () - swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix) - byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix) + results = utility.get_list(fail_tries,swift_conn,swift_container,prefix) + if not results: + return + swift_conn,objects = results + results = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix) + if not results: + return + byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj = results if encrypted: remotefiles_encr,list_enc_old = utility.list_compute_correct_names_for_encryption(objects,prefix) @@ -326,7 +335,10 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ print("Computing deletion list...") if encrypted: #update remote list with new files encrypted - swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix) + results = utility.get_list(fail_tries,swift_conn,swift_container,prefix) + if not results: + return + swift_conn,objects = results remotefiles_encr,list_enc_old = utility.list_compute_correct_names_for_encryption(objects,prefix) for o in list_enc_old: diff --git a/swift_delete_0_byte_folders.py b/swift_delete_0_byte_folders.py index 380e23b..72ec415 100644 --- a/swift_delete_0_byte_folders.py +++ b/swift_delete_0_byte_folders.py @@ -8,7 +8,10 @@ def launch(swift_container,delete,fail_tries): print ("Fail tries " + str(fail_tries)) print("___________") swift_conn = authentication.set_authentication() - swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,"") + results = utility.get_list(fail_tries,swift_conn,swift_container,"") + if not results: + return + swift_conn,objects = results remotefiles = utility.list_compute_0_byte_folders(objects) diff --git a/swift_delete_orphan_segments.py b/swift_delete_orphan_segments.py index 45569d6..1047b68 100644 --- a/swift_delete_orphan_segments.py +++ b/swift_delete_orphan_segments.py @@ -14,7 +14,10 @@ def launch(swift_container,delete,fail_tries): listfoldermanifest=[] #all files on the container - swift_conn,container_objects_raw = utility.get_list(fail_tries,swift_conn,swift_container,"") + results = utility.get_list(fail_tries,swift_conn,swift_container,"") + if not results: + return + swift_conn,container_objects_raw = results #get dict name:manifesturl only for large files byte0manifest,swift_conn,container_objects_manifest = utility.list_compute_manifest (fail_tries,container_objects_raw,swift_conn,swift_container,"") for name,manifest in container_objects_manifest.items(): @@ -31,7 +34,10 @@ def launch(swift_container,delete,fail_tries): #______________________________Folders with segments container: swift_container +"_segments" #all files on container_segments -> they are only segments - swift_conn,container_segments_objects_raw = utility.get_list(fail_tries,swift_conn,swift_container +"_segments","") + results = utility.get_list(fail_tries,swift_conn,swift_container +"_segments","") + if not results : + return + swift_conn,container_segments_objects_raw = results #get dict name:size container_segments_objects = utility.list (container_segments_objects_raw,"") #get list only with folders path name from segments diff --git a/swift_to_local.py b/swift_to_local.py index 7f539bd..b566649 100644 --- a/swift_to_local.py +++ b/swift_to_local.py @@ -33,8 +33,14 @@ def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_ print ("Files locali " + str(len(localfiles))) print("___________") swift_conn = authentication.set_authentication () - swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,prefix) - byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix) + results = utility.get_list(fail_tries,swift_conn,swift_container,prefix) + if not results: + return + swift_conn,objects = results + results = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix) + if not results: + return + byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj = results remotefiles_encr,list_enc_old = utility.list_compute_correct_names_for_encryption(objects,prefix) print ("Files remoti " + str(len(remotefiles))) diff --git a/utility.py b/utility.py index 5f2e124..6a50c15 100644 --- a/utility.py +++ b/utility.py @@ -31,8 +31,8 @@ def get_list (fail_tries,swift_conn,swift_container,prefix) : print(e) time.sleep(2) if fail_tries_counter == fail_tries - 1 : - print("Maximum tries reached. Can't download remote list for container " + swift_container + " with prefix " +prefix +". Exiting.") - sys.exit(-1) + print("Maximum tries reached. Can't download remote list for container " + swift_container + " with prefix " +prefix +".") + return False else: swift_conn = authentication.set_authentication () else : @@ -76,8 +76,8 @@ def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,pre print(e) time.sleep(2) if fail_tries_counter == fail_tries - 1 : - print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.") - sys.exit(-1) + print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +".") + return False else: swift_conn = authentication.set_authentication () else : @@ -87,7 +87,7 @@ def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,pre o["bytes"] = oheaders["content-length"] if "_xg10v10_" in oheaders["x-object-manifest"]: split1 = oheaders["x-object-manifest"].split("_xg10v10_")[0] - split2 = split1.split("/")[1] + split2 = split1.split("/")[-1] o["hash"] = split2 else: print("Impossible to get remote large file md5. Cause: Not uploaded with xgiovio method (md5 in x-object-manifest)") @@ -177,8 +177,8 @@ def listall (objects,prefix): def check_segments_size (size_limit_reading_os,size_limit_to_segment): if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16 or not is_power_of_2(size_limit_reading_os) or not is_power_of_2(size_limit_to_segment): - print ("Error size segments") - sys.exit(-2) + return False + return True def is_power_of_2 (number): result = math.log(number,2)