changed sleep to 2 sec. fixed the setting of swift_conn during deletion process. Timeout of ssl increased to 10. Added checks to segment files: they have to be power of 2.

authentication.py

@@ -11,7 +11,7 @@ swift_auth = "https://www.########"
 url = "https://########/v1/AUTH_###############"
 tok ="########"
 #######################################end authentication
-timeout = 3  #sec
+timeout = 10  #sec
 insecure = False
 
 

local_to_swift.py

@@ -299,7 +299,7 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
                     except Exception as e:
                         print("Exception during upload")
                         print(e)
-                        time.sleep(1)
+                        time.sleep(2)
                         errors_upload = errors_upload + 1
                         swift_conn = authentication.set_authentication ()
                         if fail_tries_counter == fail_tries - 1 :

settings_generate_temp_url.py

@@ -7,7 +7,7 @@ set_secretkey = False
 create_temp_url = True
 duration_in_seconds = 60*60*2
 objectpath = "#####"    #  container/object
-fail_tries = 100
+fail_tries = 30
 
 
 swift_generate_temp_url.launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpath,fail_tries)

settings_local_to_swift.py

@@ -28,7 +28,7 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s
 # 4GB 4294967296
 upload = True
 enableLarge = True
-fail_tries = 200
+fail_tries = 30
 temp_path = "\\\\?\\" + "c:\\temp\\"
 excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir"]
 delete_excluded_patterns = ["!CB_"]

settings_swift_delete_0_byte_folders.py

@@ -7,7 +7,7 @@ swift_containers = [
                     "default_segments"
                    ]
 delete = True
-fail_tries = 10
+fail_tries = 30
 
 for i in range(len(swift_containers)):
     swift_delete_0_byte_folders.launch(swift_containers[i],delete,fail_tries)

settings_swift_delete_orphan_segments.py

@@ -4,6 +4,6 @@ import swift_delete_orphan_segments
 #swift
 swift_container = "default" # container
 delete = True
-fail_tries = 10
+fail_tries = 30
 
 swift_delete_orphan_segments.launch(swift_container,delete,fail_tries)

settings_swift_to_local.py

@@ -17,7 +17,7 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s
 # 2GB 2147483648
 # 4GB 4294967296
 download = True
-fail_tries = 200
+fail_tries = 30
 excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir","!CB_"]
 batch = [
             #source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns

swift_generate_temp_url.py

@@ -20,7 +20,7 @@ def launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpat
         except Exception as e:
             print("Exception during setting / getting the secret key.")
             print(e)
-            time.sleep(1)
+            time.sleep(2)
             if fail_tries_counter == fail_tries - 1 :
                 print("Maximum tries reached. Exiting.")
                 sys.exit(-1)

swift_to_local.py

@@ -142,7 +142,7 @@ def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_
                 except Exception as e:
                     print("Exception during download")
                     print(e)
-                    time.sleep(1)
+                    time.sleep(2)
                     errors_download = errors_download + 1
                     swift_conn = authentication.set_authentication ()
                     if fail_tries_counter == fail_tries - 1 :

utility.py

@@ -3,6 +3,7 @@ __author__ = 'xgiovio'
 import authentication
 import sys, platform,os
 import time
+import math
 
 
 def get_local_files (localpath):
@@ -28,7 +29,7 @@ def get_list (fail_tries,swift_conn,swift_container,prefix) :
         except Exception as e:
             print("Exception during the download of remote list")
             print(e)
-            time.sleep(1)
+            time.sleep(2)
             if fail_tries_counter == fail_tries - 1 :
                 print("Maximum tries reached. Can't download remote list for container " + swift_container + " with prefix " +prefix +". Exiting.")
                 sys.exit(-1)
@@ -73,7 +74,7 @@ def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,pre
                     except Exception as e:
                         print("Exception during the request of metadata")
                         print(e)
-                        time.sleep(1)
+                        time.sleep(2)
                         if fail_tries_counter == fail_tries - 1 :
                             print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.")
                             sys.exit(-1)
@@ -144,7 +145,7 @@ def list_compute_manifest (fail_tries,objects,swift_conn,swift_container,prefix)
                     except Exception as e:
                         print("Exception during the request of metadata")
                         print(e)
-                        time.sleep(1)
+                        time.sleep(2)
                         if fail_tries_counter == fail_tries - 1 :
                             print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.")
                             sys.exit(-1)
@@ -175,10 +176,15 @@ def listall (objects,prefix):
     return remotefiles
 
 def check_segments_size (size_limit_reading_os,size_limit_to_segment):
-    if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16:
+    if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16 or not is_power_of_2(size_limit_reading_os) or not is_power_of_2(size_limit_to_segment):
         print ("Error size segments")
         sys.exit(-2)
 
+def is_power_of_2 (number):
+    result = math.log(number,2)
+    return (result == int(result))
+
+
 def check_segments_size_single (size_limit_reading_os):
     if size_limit_reading_os%2 != 0 or size_limit_reading_os < 16:
         print ("Error size segments")
@@ -259,24 +265,22 @@ def delete_object (swift_conn,swift_container,object,manifest,fail_tries):
                         print("Exception during deletion of manifest files")
                         print(e)
                         errors_deleting_files_of_large_obj +=1
-                        time.sleep(1)
+                        time.sleep(2)
+                        swift_conn = authentication.set_authentication()
                         if fail_tries_counter1 == fail_tries - 1 :
                             print("Maximum tries reached. Can't delete " + swift_container + "/" + object +" manifest files.Skipping")
                             skipped_files_of_large_obj += 1
-                        else:
-                            swift_conn = authentication.set_authentication ()
                     else :
                         break
         except Exception as e:
             print("Exception during deletion of file")
             print(e)
             errors_deleting_file += 1
-            time.sleep(1)
+            time.sleep(2)
+            swift_conn = authentication.set_authentication()
             if fail_tries_counter == fail_tries - 1 :
                 print("Maximum tries reached. Can't delete " + swift_container + "/" + object +".Skipping")
                 skipped_file +=1
-            else:
-                swift_conn = authentication.set_authentication ()
         else :
             break
     return [swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj]