From cc67ad764980a743f6fdc969c696894393fb963c Mon Sep 17 00:00:00 2001 From: Giovanni Di Grezia Date: Wed, 9 Dec 2015 15:39:18 +0100 Subject: [PATCH] added sync mode for local to swift, added delete function in utility, added script to remove all 0bytes directories, added script to delete large files without manifest --- local_to_swift.py | 73 +++++++++++++++---- settings_generate_temp_url.py | 8 +- settings_local_to_swift.py | 17 ++--- settings_swift_delete_0_byte_folders.py | 13 ++++ settings_swift_delete_orphan_segments.py | 6 +- settings_swift_to_local.py | 6 +- swift_delete_0_byte_folders.py | 34 +++++++++ swift_delete_orphan_segments.py | 49 ++++++++++++- ..._temp_url.py => swift_generate_temp_url.py | 0 utility.py | 22 +++++- utility_aes.py | 1 + 11 files changed, 191 insertions(+), 38 deletions(-) create mode 100644 settings_swift_delete_0_byte_folders.py create mode 100644 swift_delete_0_byte_folders.py rename generate_temp_url.py => swift_generate_temp_url.py (100%) diff --git a/local_to_swift.py b/local_to_swift.py index ba88cc7..424f900 100644 --- a/local_to_swift.py +++ b/local_to_swift.py @@ -7,7 +7,8 @@ from utility import read_in_chunks import time import shutil -def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir,delete): +def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir,delete,delete_excluded_patterns): + print ("Starting Job") print ("Localpath " + localpath) print ("Temppath " + temp_dir) print ("Swift container " + swift_container) @@ -30,10 +31,12 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ print ("Encrypted key " + "hidden") if copy_to_dir != None: print ("Copy to dir " + copy_to_dir) + print ("Delete files on dest if not present on source " + str(delete)) print("___________") + #checks utility.check_segments_size (size_limit_reading_os,size_limit_to_segment) if utility.check_start_slash(swift_container) or utility.check_start_slash(prefix) or not utility.check_end_slash(localpath) or not utility.check_end_slash(temp_dir) or utility.check_end_slash(swift_container) or not utility.check_end_slash(prefix): @@ -318,6 +321,7 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ dellist = [] + delete_skipped_filters = 0 print("") print("Computing deletion list...") if encrypted: @@ -328,28 +332,67 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_ for o in list_enc_old: dellist.append(o) for rname in remotefiles_encr.keys(): - if rname.endswith("_xg10v10_encrypted"): - rname_pure = rname.split("_xg10v10_encrypted")[0] - if rname_pure in remotefiles_encr.keys(): - dellist.append(remotefiles_encr[rname_pure]) - if rname_pure not in localfiles.keys(): - dellist.append(remotefiles_encr[rname]) - else: - if rname not in localfiles.keys(): - dellist.append(remotefiles_encr[rname]) + delete_file = True + for pattern in delete_excluded_patterns: + if pattern in rname: + delete_file = False + print("Skipped " + rname + " due to filters : " + pattern ) + delete_skipped_filters +=1 + break + + if delete_file : + if rname.endswith("_xg10v10_encrypted"): + rname_pure = rname.split("_xg10v10_encrypted")[0] + if rname_pure in remotefiles_encr.keys(): + dellist.append(remotefiles_encr[rname_pure]) + if rname_pure not in localfiles.keys(): + dellist.append(remotefiles_encr[rname]) + else: + if rname not in localfiles.keys(): + dellist.append(remotefiles_encr[rname]) else: for rname in remotefiles.keys(): - if rname not in localfiles.keys(): - dellist.append(rname) + delete_file = True + for pattern in delete_excluded_patterns: + if pattern in rname: + delete_file = False + print("Skipped " + rname + " due to filters : " + pattern ) + delete_skipped_filters +=1 + break + if delete_file: + if rname not in localfiles.keys(): + dellist.append(rname) print("___________Files to delete______") for files in dellist: print(files) print("___________") - if delete : + print("Files to delete " + str(len(dellist))) + print("Skipped files due to filters " + str(delete_skipped_filters)) + + if delete and len(dellist) > 0: + errors_deleting_files = 0 + errors_deleting_files_of_large_objs = 0 + skipped_files = 0 + skipped_files_of_large_objs = 0 for object in dellist: - swift_conn = utility.delete_object(swift_conn,swift_container,prefix + object,remotefiles_xobj[object],fail_tries) + swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj = utility.delete_object(swift_conn,swift_container,prefix + object,remotefiles_xobj[object],fail_tries) + errors_deleting_files += errors_deleting_file + errors_deleting_files_of_large_objs += errors_deleting_files_of_large_obj + skipped_files += skipped_file + skipped_files_of_large_objs += skipped_files_of_large_obj + print("___________") + print("Errors deleting files " + str(errors_deleting_files)) + print("Errors deleting files of large objs " + str(errors_deleting_files_of_large_objs)) + print("Skipped files to delete " + str(skipped_files)) + print("Skipped files of large objs to delete " + str(skipped_files_of_large_objs)) else: - print("Delete disabled") + if not delete: + print("Delete disabled") swift_conn.close() + print("") + print("End Job") + print("___________") + print("___________") + print("___________") diff --git a/settings_generate_temp_url.py b/settings_generate_temp_url.py index bdc1a92..4593103 100644 --- a/settings_generate_temp_url.py +++ b/settings_generate_temp_url.py @@ -1,13 +1,13 @@ __author__ = 'xgiovio' -import generate_temp_url +import swift_generate_temp_url secretkey ="" set_secretkey = False create_temp_url = True -duration_in_seconds = 60*60*24*7 -objectpath = "/########" # container/object +duration_in_seconds = 60*60*2 +objectpath = "#####" # container/object fail_tries = 100 -generate_temp_url.launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpath,fail_tries) \ No newline at end of file +swift_generate_temp_url.launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpath,fail_tries) \ No newline at end of file diff --git a/settings_local_to_swift.py b/settings_local_to_swift.py index ff53d80..49c6ba4 100644 --- a/settings_local_to_swift.py +++ b/settings_local_to_swift.py @@ -26,21 +26,20 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s # 1GB 1073741824 # 2GB 2147483648 # 4GB 4294967296 -upload = False +upload = True enableLarge = True -delete_remote_files_inexistent_on_source = True -fail_tries = 99999999999 +fail_tries = 200 temp_path = "\\\\?\\" + "c:\\temp\\" excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir"] +delete_excluded_patterns = ["!CB_"] batch = [ - #source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns,copy_to_dir - ["\\\\?\\" + "c:\\orig\\","default","prefix/",False,False,None,[],None] , - ["\\\\?\\" + "c:\\orig2\\","default","prefix/",False,False,None,[],None] , - #["\\\\?\\" + "c:\\orig3\\","default","prefix/",False,False,None,[],None] , + #source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns,copy_to_dir,delete_inexistent_remote_files,additional_delete_excluded_patterns + ["\\\\?\\" + "c:\\test\\","default","test/",False,True,"ciao",[],None,True,[]] , + ["\\\\?\\" + "c:\\test\\","default","test/",False,True,"ciao",[],None,True,[]] , ] ############################################################################################### for job in batch: - #local folder,temp path, swift container, swift prefix, size to segment, size reading limit os, upload enabled?. upload large enabled? , fail tries, md5 comparison enabled?, encrypted?, encryption_key,additional_excluded_patterns,copy_to_dir, delete_inexistent_remote_files - local_to_swift.launch(job[0],temp_path,job[1],job[2],size_limit_to_segment,size_limit_reading_os,upload,enableLarge,fail_tries, job[3],job[4],job[5], job[6] + excluded_patterns,job[7],delete_remote_files_inexistent_on_source) \ No newline at end of file + #local folder,temp path, swift container, swift prefix, size to segment, size reading limit os, upload enabled?. upload large enabled? , fail tries, md5 comparison enabled?, encrypted?, encryption_key,additional_excluded_patterns,copy_to_dir, delete_inexistent_remote_files,additional_delete_excluded_patterns + local_to_swift.launch(job[0],temp_path,job[1],job[2],size_limit_to_segment,size_limit_reading_os,upload,enableLarge,fail_tries, job[3],job[4],job[5], job[6] + excluded_patterns,job[7],job[8],job[9] + delete_excluded_patterns) \ No newline at end of file diff --git a/settings_swift_delete_0_byte_folders.py b/settings_swift_delete_0_byte_folders.py new file mode 100644 index 0000000..10c4e55 --- /dev/null +++ b/settings_swift_delete_0_byte_folders.py @@ -0,0 +1,13 @@ +__author__ = 'xgiovio' + +import swift_delete_0_byte_folders +#swift +swift_containers = [ + "default", + "default_segments" + ] +delete = True +fail_tries = 10 + +for i in range(len(swift_containers)): + swift_delete_0_byte_folders.launch(swift_containers[i],delete,fail_tries) diff --git a/settings_swift_delete_orphan_segments.py b/settings_swift_delete_orphan_segments.py index 57d5c03..bb90262 100644 --- a/settings_swift_delete_orphan_segments.py +++ b/settings_swift_delete_orphan_segments.py @@ -2,8 +2,8 @@ __author__ = 'xgiovio' import swift_delete_orphan_segments #swift -swift_container = "####" # container -delete = False -fail_tries = 100 +swift_container = "default" # container +delete = True +fail_tries = 10 swift_delete_orphan_segments.launch(swift_container,delete,fail_tries) diff --git a/settings_swift_to_local.py b/settings_swift_to_local.py index ae037c1..4c94ba9 100644 --- a/settings_swift_to_local.py +++ b/settings_swift_to_local.py @@ -17,11 +17,11 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s # 2GB 2147483648 # 4GB 4294967296 download = True -fail_tries = 1000000 -excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir"] +fail_tries = 200 +excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir","!CB_"] batch = [ #source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns - ["\\\\?\\" + "C:\\test\\","default","test/",False,True,"pass",[]] , + ["\\\\?\\" + "C:\\test\\","default","test/",False,True,"ciao",[]] , ] diff --git a/swift_delete_0_byte_folders.py b/swift_delete_0_byte_folders.py new file mode 100644 index 0000000..380e23b --- /dev/null +++ b/swift_delete_0_byte_folders.py @@ -0,0 +1,34 @@ + +import authentication, utility + +def launch(swift_container,delete,fail_tries): + + print ("Swift container " + swift_container) + print ("Delete " + str(delete)) + print ("Fail tries " + str(fail_tries)) + print("___________") + swift_conn = authentication.set_authentication() + swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,"") + remotefiles = utility.list_compute_0_byte_folders(objects) + + + for file in remotefiles: + print("0 byte directory to delete: " + file ) + print("___________") + print("Directories to delete " + str(len(remotefiles)) ) + + if delete and len(remotefiles) > 0: + errors_deleting_files = 0 + skipped_files = 0 + for file in remotefiles: + swift_conn,errors_deleting_file,_,skipped_file,_ = utility.delete_object(swift_conn,swift_container,file,None,fail_tries) + errors_deleting_files += errors_deleting_file + skipped_files += skipped_file + print("___________") + print("Errors deleting files " + str(errors_deleting_files)) + print("Skipped files to delete " + str(skipped_files)) + else: + if not delete and len(remotefiles) > 0: + print("Delete Disabled") + print("___________") + print("___________") diff --git a/swift_delete_orphan_segments.py b/swift_delete_orphan_segments.py index 6f7aff1..45569d6 100644 --- a/swift_delete_orphan_segments.py +++ b/swift_delete_orphan_segments.py @@ -6,6 +6,8 @@ import authentication, utility def launch(swift_container,delete,fail_tries): print ("Swift container " + swift_container) + print ("Delete " + str(delete)) + print ("Fail tries " + str(fail_tries)) print("___________") swift_conn = authentication.set_authentication () @@ -24,8 +26,9 @@ def launch(swift_container,delete,fail_tries): #_____________________________________________________________________________________________________________________ #______________________________Folders with segments listfoldersegmentsprefix = [] + listfoldersegmentsprefixfull = {} - #______________________________Folders with segments : container: swift_container +"_segments" + #______________________________Folders with segments container: swift_container +"_segments" #all files on container_segments -> they are only segments swift_conn,container_segments_objects_raw = utility.get_list(fail_tries,swift_conn,swift_container +"_segments","") @@ -33,52 +36,87 @@ def launch(swift_container,delete,fail_tries): container_segments_objects = utility.list (container_segments_objects_raw,"") #get list only with folders path name from segments listfolder = [] + for name in container_segments_objects.keys(): + if (swift_container +"_segments/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys(): + listfoldersegmentsprefixfull[swift_container +"_segments/" + utility.folder_from_path(name,"/")]= [] + listfoldersegmentsprefixfull[swift_container +"_segments/" + utility.folder_from_path(name,"/")].append(swift_container +"_segments/" + name) + if utility.folder_from_path(name,"/") not in listfolder : listfolder.append(utility.folder_from_path(name,"/")) for o in listfolder: listfoldersegmentsprefix.append(swift_container +"_segments/" + o) - #______________________________Folders with segments : container: swift_container path: @SynologyCloudSync/ + #______________________________Folders with segments container: swift_container path: @SynologyCloudSync/ #all files @SynologyCloudSync/ on container -> they are only segments #get dict name:size container_segments_objects = utility.filter_list_begin(container_objects_raw,"@SynologyCloudSync/","") #get list only with folders path name from segments listfolder = [] for name in container_segments_objects.keys(): + if (swift_container +"/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys(): + listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")]=[] + listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")].append(swift_container +"/" + name) if utility.folder_from_path(name,"/") not in listfolder : listfolder.append(utility.folder_from_path(name,"/")) for o in listfolder: listfoldersegmentsprefix.append(swift_container +"/" + o) - #______________________________Folders with segments "!CB_" + #______________________________Folders with segments container: swift_container path: files with !CB_ #all files !CB_ of CloudBerry on container -> they are only segments #get dict name:size container_segments_objects = utility.search_list(container_objects_raw,"!CB_","") #get list only with folders path name from segments listfolder = [] for name in container_segments_objects.keys(): + if (swift_container +"/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys(): + listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")]=[] + listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")].append(swift_container +"/" + name) if utility.folder_from_path(name,"_") not in listfolder : listfolder.append(utility.folder_from_path(name,"_")) for o in listfolder: listfoldersegmentsprefix.append(swift_container +"/" + o) + + #_____________________________________________________________________________________________________________________ ''' for o in listfoldersegmentsprefix: print(o) ''' #_____________________________________________________________________________________________________________________ + print("___________") segments_to_delete = [] segments_not_listed = [] for manifesturlsegments in listfoldersegmentsprefix : if manifesturlsegments not in listfoldermanifest: segments_to_delete.append(manifesturlsegments) + print("Segments folders to delete: " + str(len(segments_to_delete))) for manifest in segments_to_delete: print("Segments folder to delete: " + manifest) + print("___________") + if delete: + errors_deleting_files = 0 + skipped_files = 0 + for short,longlist in listfoldersegmentsprefixfull.items(): + if short in segments_to_delete: + for long in longlist: + del_container = long.split("/")[0] + del_object = long.replace(del_container + "/","") + swift_conn,errors_deleting_file,_,skipped_file,_ = utility.delete_object(swift_conn,del_container,del_object,None,fail_tries) + errors_deleting_files += errors_deleting_file + skipped_files += skipped_file + print("___________") + print("Errors deleting files " + str(errors_deleting_files)) + print("Skipped files to delete " + str(skipped_files)) + else: + print("Delete Disabled") + + print("___________") + #_____ for manifesturl in listfoldermanifest : if manifesturl not in listfoldersegmentsprefix: segments_not_listed.append(manifesturl) @@ -86,4 +124,9 @@ def launch(swift_container,delete,fail_tries): for manifest in segments_not_listed: print("Segments not present in given segments list : " + manifest) + #_____ + + + + swift_conn.close() diff --git a/generate_temp_url.py b/swift_generate_temp_url.py similarity index 100% rename from generate_temp_url.py rename to swift_generate_temp_url.py diff --git a/utility.py b/utility.py index f6ec980..096c4d8 100644 --- a/utility.py +++ b/utility.py @@ -234,6 +234,10 @@ def set_dash(): def delete_object (swift_conn,swift_container,object,manifest,fail_tries): + errors_deleting_file = 0 + errors_deleting_files_of_large_obj = 0 + skipped_file = 0 + skipped_files_of_large_obj = 0 for fail_tries_counter in range (fail_tries) : try: print("Deleting " + swift_container + "/" + object) @@ -254,9 +258,11 @@ def delete_object (swift_conn,swift_container,object,manifest,fail_tries): except Exception as e: print("Exception during deletion of manifest files") print(e) + errors_deleting_files_of_large_obj +=1 time.sleep(1) if fail_tries_counter1 == fail_tries - 1 : print("Maximum tries reached. Can't delete " + swift_container + "/" + object +" manifest files.Skipping") + skipped_files_of_large_obj += 1 else: swift_conn = authentication.set_authentication () else : @@ -264,12 +270,26 @@ def delete_object (swift_conn,swift_container,object,manifest,fail_tries): except Exception as e: print("Exception during deletion of file") print(e) + errors_deleting_file += 1 time.sleep(1) if fail_tries_counter == fail_tries - 1 : print("Maximum tries reached. Can't delete " + swift_container + "/" + object +".Skipping") + skipped_file +=1 else: swift_conn = authentication.set_authentication () else : break - return swift_conn + return [swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj] + + +def list_compute_0_byte_folders (objects): + + remotefiles = [] + + for o in objects : + if o["content_type"] == "application/directory" and int(o["bytes"]) == 0: + remotefiles.append(o["name"]) + + return remotefiles + diff --git a/utility_aes.py b/utility_aes.py index 286ecb8..9c85205 100644 --- a/utility_aes.py +++ b/utility_aes.py @@ -1,6 +1,7 @@ import hashlib,os from utility import read_in_chunks + try: pycrypto = True from Crypto.Cipher import AES