From 77f3a9fa053cf6e302f2d21d76b2a70be4356a23 Mon Sep 17 00:00:00 2001 From: Giovanni Di Grezia Date: Sun, 8 May 2016 22:46:08 +0200 Subject: [PATCH] changed sleep to 2 sec. fixed the setting of swift_conn during deletion process. Timeout of ssl increased to 10. Added checks to segment files: they have to be power of 2. --- authentication.py | 2 +- local_to_swift.py | 2 +- settings_generate_temp_url.py | 2 +- settings_local_to_swift.py | 2 +- settings_swift_delete_0_byte_folders.py | 2 +- settings_swift_delete_orphan_segments.py | 2 +- settings_swift_to_local.py | 2 +- swift_generate_temp_url.py | 2 +- swift_to_local.py | 2 +- utility.py | 24 ++++++++++++++---------- 10 files changed, 23 insertions(+), 19 deletions(-) diff --git a/authentication.py b/authentication.py index 5e0a240..bfbea93 100644 --- a/authentication.py +++ b/authentication.py @@ -11,7 +11,7 @@ swift_auth = "https://www.########" url = "https://########/v1/AUTH_###############" tok ="########" #######################################end authentication -timeout = 3 #sec +timeout = 10 #sec insecure = False diff --git a/local_to_swift.py b/local_to_swift.py index 0e9aa16..f37f0a1 100644 --- a/local_to_swift.py +++ b/local_to_swift.py @@ -299,7 +299,7 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ except Exception as e: print("Exception during upload") print(e) - time.sleep(1) + time.sleep(2) errors_upload = errors_upload + 1 swift_conn = authentication.set_authentication () if fail_tries_counter == fail_tries - 1 : diff --git a/settings_generate_temp_url.py b/settings_generate_temp_url.py index 4593103..e483e94 100644 --- a/settings_generate_temp_url.py +++ b/settings_generate_temp_url.py @@ -7,7 +7,7 @@ set_secretkey = False create_temp_url = True duration_in_seconds = 60*60*2 objectpath = "#####" # container/object -fail_tries = 100 +fail_tries = 30 swift_generate_temp_url.launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpath,fail_tries) \ No newline at end of file diff --git a/settings_local_to_swift.py b/settings_local_to_swift.py index 49c6ba4..919d427 100644 --- a/settings_local_to_swift.py +++ b/settings_local_to_swift.py @@ -28,7 +28,7 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s # 4GB 4294967296 upload = True enableLarge = True -fail_tries = 200 +fail_tries = 30 temp_path = "\\\\?\\" + "c:\\temp\\" excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir"] delete_excluded_patterns = ["!CB_"] diff --git a/settings_swift_delete_0_byte_folders.py b/settings_swift_delete_0_byte_folders.py index 10c4e55..41f8f3c 100644 --- a/settings_swift_delete_0_byte_folders.py +++ b/settings_swift_delete_0_byte_folders.py @@ -7,7 +7,7 @@ swift_containers = [ "default_segments" ] delete = True -fail_tries = 10 +fail_tries = 30 for i in range(len(swift_containers)): swift_delete_0_byte_folders.launch(swift_containers[i],delete,fail_tries) diff --git a/settings_swift_delete_orphan_segments.py b/settings_swift_delete_orphan_segments.py index bb90262..bfd19a0 100644 --- a/settings_swift_delete_orphan_segments.py +++ b/settings_swift_delete_orphan_segments.py @@ -4,6 +4,6 @@ import swift_delete_orphan_segments #swift swift_container = "default" # container delete = True -fail_tries = 10 +fail_tries = 30 swift_delete_orphan_segments.launch(swift_container,delete,fail_tries) diff --git a/settings_swift_to_local.py b/settings_swift_to_local.py index 4c94ba9..2c62980 100644 --- a/settings_swift_to_local.py +++ b/settings_swift_to_local.py @@ -17,7 +17,7 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s # 2GB 2147483648 # 4GB 4294967296 download = True -fail_tries = 200 +fail_tries = 30 excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir","!CB_"] batch = [ #source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns diff --git a/swift_generate_temp_url.py b/swift_generate_temp_url.py index 17f06de..df612eb 100644 --- a/swift_generate_temp_url.py +++ b/swift_generate_temp_url.py @@ -20,7 +20,7 @@ def launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpat except Exception as e: print("Exception during setting / getting the secret key.") print(e) - time.sleep(1) + time.sleep(2) if fail_tries_counter == fail_tries - 1 : print("Maximum tries reached. Exiting.") sys.exit(-1) diff --git a/swift_to_local.py b/swift_to_local.py index 98d4b73..7f539bd 100644 --- a/swift_to_local.py +++ b/swift_to_local.py @@ -142,7 +142,7 @@ def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_ except Exception as e: print("Exception during download") print(e) - time.sleep(1) + time.sleep(2) errors_download = errors_download + 1 swift_conn = authentication.set_authentication () if fail_tries_counter == fail_tries - 1 : diff --git a/utility.py b/utility.py index 096c4d8..5f2e124 100644 --- a/utility.py +++ b/utility.py @@ -3,6 +3,7 @@ __author__ = 'xgiovio' import authentication import sys, platform,os import time +import math def get_local_files (localpath): @@ -28,7 +29,7 @@ def get_list (fail_tries,swift_conn,swift_container,prefix) : except Exception as e: print("Exception during the download of remote list") print(e) - time.sleep(1) + time.sleep(2) if fail_tries_counter == fail_tries - 1 : print("Maximum tries reached. Can't download remote list for container " + swift_container + " with prefix " +prefix +". Exiting.") sys.exit(-1) @@ -73,7 +74,7 @@ def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,pre except Exception as e: print("Exception during the request of metadata") print(e) - time.sleep(1) + time.sleep(2) if fail_tries_counter == fail_tries - 1 : print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.") sys.exit(-1) @@ -144,7 +145,7 @@ def list_compute_manifest (fail_tries,objects,swift_conn,swift_container,prefix) except Exception as e: print("Exception during the request of metadata") print(e) - time.sleep(1) + time.sleep(2) if fail_tries_counter == fail_tries - 1 : print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.") sys.exit(-1) @@ -175,10 +176,15 @@ def listall (objects,prefix): return remotefiles def check_segments_size (size_limit_reading_os,size_limit_to_segment): - if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16: + if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16 or not is_power_of_2(size_limit_reading_os) or not is_power_of_2(size_limit_to_segment): print ("Error size segments") sys.exit(-2) +def is_power_of_2 (number): + result = math.log(number,2) + return (result == int(result)) + + def check_segments_size_single (size_limit_reading_os): if size_limit_reading_os%2 != 0 or size_limit_reading_os < 16: print ("Error size segments") @@ -259,24 +265,22 @@ def delete_object (swift_conn,swift_container,object,manifest,fail_tries): print("Exception during deletion of manifest files") print(e) errors_deleting_files_of_large_obj +=1 - time.sleep(1) + time.sleep(2) + swift_conn = authentication.set_authentication() if fail_tries_counter1 == fail_tries - 1 : print("Maximum tries reached. Can't delete " + swift_container + "/" + object +" manifest files.Skipping") skipped_files_of_large_obj += 1 - else: - swift_conn = authentication.set_authentication () else : break except Exception as e: print("Exception during deletion of file") print(e) errors_deleting_file += 1 - time.sleep(1) + time.sleep(2) + swift_conn = authentication.set_authentication() if fail_tries_counter == fail_tries - 1 : print("Maximum tries reached. Can't delete " + swift_container + "/" + object +".Skipping") skipped_file +=1 - else: - swift_conn = authentication.set_authentication () else : break return [swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj]