xgiovio/openstack-swift-backup-restore-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added sync mode for local to swift, added delete function in utility, added script to remove all 0bytes directories, added script to delete large files without manifest

Browse Source
This commit is contained in:
Giovanni Di Grezia
2015-12-09 15:39:18 +01:00
parent 07972c55c5
commit cc67ad7649
11 changed files with 191 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
swift_delete_orphan_segments.py
View File

@@ -6,6 +6,8 @@ import authentication, utility
def launch(swift_container,delete,fail_tries):
print ("Swift container " + swift_container)
print ("Delete " + str(delete))
print ("Fail tries " + str(fail_tries))
print("___________")
swift_conn = authentication.set_authentication ()
@@ -24,8 +26,9 @@ def launch(swift_container,delete,fail_tries):
#_____________________________________________________________________________________________________________________
#______________________________Folders with segments
listfoldersegmentsprefix = []
listfoldersegmentsprefixfull = {}
#______________________________Folders with segments : container: swift_container +"_segments"
#______________________________Folders with segments container: swift_container +"_segments"
#all files on container_segments -> they are only segments
swift_conn,container_segments_objects_raw = utility.get_list(fail_tries,swift_conn,swift_container +"_segments","")
@@ -33,52 +36,87 @@ def launch(swift_container,delete,fail_tries):
container_segments_objects = utility.list (container_segments_objects_raw,"")
#get list only with folders path name from segments
listfolder = []
for name in container_segments_objects.keys():
if (swift_container +"_segments/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys():
listfoldersegmentsprefixfull[swift_container +"_segments/" + utility.folder_from_path(name,"/")]= []
listfoldersegmentsprefixfull[swift_container +"_segments/" + utility.folder_from_path(name,"/")].append(swift_container +"_segments/" + name)
if utility.folder_from_path(name,"/") not in listfolder :
listfolder.append(utility.folder_from_path(name,"/"))
for o in listfolder:
listfoldersegmentsprefix.append(swift_container +"_segments/" + o)
#______________________________Folders with segments : container: swift_container path: @SynologyCloudSync/
#______________________________Folders with segments container: swift_container path: @SynologyCloudSync/
#all files @SynologyCloudSync/ on container -> they are only segments
#get dict name:size
container_segments_objects = utility.filter_list_begin(container_objects_raw,"@SynologyCloudSync/","")
#get list only with folders path name from segments
listfolder = []
for name in container_segments_objects.keys():
if (swift_container +"/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys():
listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")]=[]
listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")].append(swift_container +"/" + name)
if utility.folder_from_path(name,"/") not in listfolder :
listfolder.append(utility.folder_from_path(name,"/"))
for o in listfolder:
listfoldersegmentsprefix.append(swift_container +"/" + o)
#______________________________Folders with segments "!CB_"
#______________________________Folders with segments container: swift_container path: files with !CB_
#all files !CB_ of CloudBerry on container -> they are only segments
#get dict name:size
container_segments_objects = utility.search_list(container_objects_raw,"!CB_","")
#get list only with folders path name from segments
listfolder = []
for name in container_segments_objects.keys():
if (swift_container +"/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys():
listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")]=[]
listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")].append(swift_container +"/" + name)
if utility.folder_from_path(name,"_") not in listfolder :
listfolder.append(utility.folder_from_path(name,"_"))
for o in listfolder:
listfoldersegmentsprefix.append(swift_container +"/" + o)
#_____________________________________________________________________________________________________________________
'''
for o in listfoldersegmentsprefix:
print(o)
'''
#_____________________________________________________________________________________________________________________
print("___________")
segments_to_delete = []
segments_not_listed = []
for manifesturlsegments in listfoldersegmentsprefix :
if manifesturlsegments not in listfoldermanifest:
segments_to_delete.append(manifesturlsegments)
print("Segments folders to delete: " + str(len(segments_to_delete)))
for manifest in segments_to_delete:
print("Segments folder to delete: " + manifest)
print("___________")
if delete:
errors_deleting_files = 0
skipped_files = 0
for short,longlist in listfoldersegmentsprefixfull.items():
if short in segments_to_delete:
for long in longlist:
del_container = long.split("/")[0]
del_object = long.replace(del_container + "/","")
swift_conn,errors_deleting_file,_,skipped_file,_ = utility.delete_object(swift_conn,del_container,del_object,None,fail_tries)
errors_deleting_files += errors_deleting_file
skipped_files += skipped_file
print("___________")
print("Errors deleting files " + str(errors_deleting_files))
print("Skipped files to delete " + str(skipped_files))
else:
print("Delete Disabled")
print("___________")
#_____
for manifesturl in listfoldermanifest :
if manifesturl not in listfoldersegmentsprefix:
segments_not_listed.append(manifesturl)
@@ -86,4 +124,9 @@ def launch(swift_container,delete,fail_tries):
for manifest in segments_not_listed:
print("Segments not present in given segments list : " + manifest)
#_____
swift_conn.close()