added sync mode for local to swift, added delete function in utility, added script to remove all 0bytes directories, added script to delete large files without manifest

local_to_swift.py

@@ -7,7 +7,8 @@ from utility import read_in_chunks
 import time
 import shutil
 
-def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir,delete):
+def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_limit_reading_os,upload,uploadlarge,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns,copy_to_dir,delete,delete_excluded_patterns):
+    print ("Starting Job")
     print ("Localpath " + localpath)
     print ("Temppath " + temp_dir)
     print ("Swift container " + swift_container)
@@ -30,10 +31,12 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
             print ("Encrypted key " + "hidden")
     if copy_to_dir != None:
         print ("Copy to dir " + copy_to_dir)
+    print ("Delete files on dest if not present on source " + str(delete))
     print("___________")
 
 
 
+
     #checks
     utility.check_segments_size (size_limit_reading_os,size_limit_to_segment)
     if  utility.check_start_slash(swift_container) or  utility.check_start_slash(prefix) or not utility.check_end_slash(localpath) or not utility.check_end_slash(temp_dir) or  utility.check_end_slash(swift_container) or not utility.check_end_slash(prefix):
@@ -318,6 +321,7 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
 
 
     dellist = []
+    delete_skipped_filters = 0
     print("")
     print("Computing deletion list...")
     if encrypted:
@@ -328,28 +332,67 @@ def launch(localpath,temp_dir,swift_container,prefix,size_limit_to_segment,size_
         for o in list_enc_old:
            dellist.append(o)
         for rname in remotefiles_encr.keys():
-            if rname.endswith("_xg10v10_encrypted"):
+            delete_file = True
-                rname_pure = rname.split("_xg10v10_encrypted")[0]
+            for pattern in delete_excluded_patterns:
-                if rname_pure in remotefiles_encr.keys():
+                if pattern in rname:
-                   dellist.append(remotefiles_encr[rname_pure])
+                    delete_file = False
-                if rname_pure not in localfiles.keys():
+                    print("Skipped " + rname + " due to filters : " + pattern )
-                   dellist.append(remotefiles_encr[rname])
+                    delete_skipped_filters +=1
-            else:
+                    break
-                if rname not in localfiles.keys():
+
-                    dellist.append(remotefiles_encr[rname])
+            if delete_file :
+                if rname.endswith("_xg10v10_encrypted"):
+                    rname_pure = rname.split("_xg10v10_encrypted")[0]
+                    if rname_pure in remotefiles_encr.keys():
+                       dellist.append(remotefiles_encr[rname_pure])
+                    if rname_pure not in localfiles.keys():
+                       dellist.append(remotefiles_encr[rname])
+                else:
+                    if rname not in localfiles.keys():
+                        dellist.append(remotefiles_encr[rname])
     else:
         for rname in remotefiles.keys():
-            if rname not in localfiles.keys():
+            delete_file = True
-                dellist.append(rname)
+            for pattern in delete_excluded_patterns:
+                if pattern in rname:
+                    delete_file = False
+                    print("Skipped " + rname + " due to filters : " + pattern )
+                    delete_skipped_filters +=1
+                    break
+            if delete_file:
+                if rname not in localfiles.keys():
+                    dellist.append(rname)
 
     print("___________Files to delete______")
     for files in dellist:
         print(files)
     print("___________")
-    if delete :
+    print("Files to delete " + str(len(dellist)))
+    print("Skipped files due to filters " + str(delete_skipped_filters))
+
+    if delete and len(dellist) > 0:
+        errors_deleting_files = 0
+        errors_deleting_files_of_large_objs = 0
+        skipped_files = 0
+        skipped_files_of_large_objs = 0
         for object in dellist:
-            swift_conn = utility.delete_object(swift_conn,swift_container,prefix + object,remotefiles_xobj[object],fail_tries)
+            swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj = utility.delete_object(swift_conn,swift_container,prefix + object,remotefiles_xobj[object],fail_tries)
+            errors_deleting_files += errors_deleting_file
+            errors_deleting_files_of_large_objs += errors_deleting_files_of_large_obj
+            skipped_files += skipped_file
+            skipped_files_of_large_objs += skipped_files_of_large_obj
+        print("___________")
+        print("Errors deleting files " + str(errors_deleting_files))
+        print("Errors deleting files of large objs " + str(errors_deleting_files_of_large_objs))
+        print("Skipped files to delete " + str(skipped_files))
+        print("Skipped files of large objs to delete " + str(skipped_files_of_large_objs))
     else:
-        print("Delete disabled")
+        if not delete:
+            print("Delete disabled")
 
     swift_conn.close()
+    print("")
+    print("End Job")
+    print("___________")
+    print("___________")
+    print("___________")

settings_generate_temp_url.py

@@ -1,13 +1,13 @@
 __author__ = 'xgiovio'
 
-import generate_temp_url
+import swift_generate_temp_url
 
 secretkey =""
 set_secretkey = False
 create_temp_url = True
-duration_in_seconds = 60*60*24*7
+duration_in_seconds = 60*60*2
-objectpath = "/########"    #  container/object
+objectpath = "#####"    #  container/object
 fail_tries = 100
 
 
-generate_temp_url.launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpath,fail_tries)
+swift_generate_temp_url.launch(secretkey,set_secretkey,create_temp_url,duration_in_seconds,objectpath,fail_tries)

settings_local_to_swift.py

@@ -26,21 +26,20 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s
 # 1GB 1073741824
 # 2GB 2147483648
 # 4GB 4294967296
-upload = False
+upload = True
 enableLarge = True
-delete_remote_files_inexistent_on_source = True
+fail_tries = 200
-fail_tries = 99999999999
 temp_path = "\\\\?\\" + "c:\\temp\\"
 excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir"]
+delete_excluded_patterns = ["!CB_"]
 batch = [
-            #source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns,copy_to_dir
+            #source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns,copy_to_dir,delete_inexistent_remote_files,additional_delete_excluded_patterns
-            ["\\\\?\\" + "c:\\orig\\","default","prefix/",False,False,None,[],None] ,
-            ["\\\\?\\" + "c:\\orig2\\","default","prefix/",False,False,None,[],None] ,
-            #["\\\\?\\" + "c:\\orig3\\","default","prefix/",False,False,None,[],None] ,
 
+            ["\\\\?\\" + "c:\\test\\","default","test/",False,True,"ciao",[],None,True,[]] ,
+            ["\\\\?\\" + "c:\\test\\","default","test/",False,True,"ciao",[],None,True,[]] ,
         ]
 
 ###############################################################################################
 for job in batch:
-    #local folder,temp path, swift container, swift prefix, size to segment, size reading limit os, upload enabled?. upload large enabled? , fail tries, md5 comparison enabled?, encrypted?, encryption_key,additional_excluded_patterns,copy_to_dir, delete_inexistent_remote_files
+    #local folder,temp path, swift container, swift prefix, size to segment, size reading limit os, upload enabled?. upload large enabled? , fail tries, md5 comparison enabled?, encrypted?, encryption_key,additional_excluded_patterns,copy_to_dir, delete_inexistent_remote_files,additional_delete_excluded_patterns
-    local_to_swift.launch(job[0],temp_path,job[1],job[2],size_limit_to_segment,size_limit_reading_os,upload,enableLarge,fail_tries, job[3],job[4],job[5], job[6] + excluded_patterns,job[7],delete_remote_files_inexistent_on_source)
+    local_to_swift.launch(job[0],temp_path,job[1],job[2],size_limit_to_segment,size_limit_reading_os,upload,enableLarge,fail_tries, job[3],job[4],job[5], job[6] + excluded_patterns,job[7],job[8],job[9] + delete_excluded_patterns)

settings_swift_delete_0_byte_folders.py

@@ -0,0 +1,13 @@
+__author__ = 'xgiovio'
+
+import swift_delete_0_byte_folders
+#swift
+swift_containers = [
+                    "default",
+                    "default_segments"
+                   ]
+delete = True
+fail_tries = 10
+
+for i in range(len(swift_containers)):
+    swift_delete_0_byte_folders.launch(swift_containers[i],delete,fail_tries)

settings_swift_delete_orphan_segments.py

@@ -2,8 +2,8 @@ __author__ = 'xgiovio'
 
 import swift_delete_orphan_segments
 #swift
-swift_container = "####" # container
+swift_container = "default" # container
-delete = False
+delete = True
-fail_tries = 100
+fail_tries = 10
 
 swift_delete_orphan_segments.launch(swift_container,delete,fail_tries)

settings_swift_to_local.py

@@ -17,11 +17,11 @@ size_limit_reading_os = 134217728 #must be a power of 2 and smaller/equal than s
 # 2GB 2147483648
 # 4GB 4294967296
 download = True
-fail_tries = 1000000
+fail_tries = 200
-excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir"]
+excluded_patterns = ["Thumbs.db",".DS_Store","_gsdata_","__MACOSX", "desktop.ini","@eaDir","!CB_"]
 batch = [
             #source, swift container, swift prefix, md5 comparison enabled?, encrypted?, encryption_key, additional_excluded_patterns
-            ["\\\\?\\" + "C:\\test\\","default","test/",False,True,"pass",[]] ,
+            ["\\\\?\\" + "C:\\test\\","default","test/",False,True,"ciao",[]] ,
 
         ]
 

swift_delete_0_byte_folders.py

@@ -0,0 +1,34 @@
+
+import authentication, utility
+
+def launch(swift_container,delete,fail_tries):
+
+    print ("Swift container " + swift_container)
+    print ("Delete " + str(delete))
+    print ("Fail tries " + str(fail_tries))
+    print("___________")
+    swift_conn = authentication.set_authentication()
+    swift_conn,objects = utility.get_list(fail_tries,swift_conn,swift_container,"")
+    remotefiles = utility.list_compute_0_byte_folders(objects)
+
+
+    for file in remotefiles:
+        print("0 byte directory to delete: " + file )
+    print("___________")
+    print("Directories to delete " + str(len(remotefiles)) )
+
+    if delete and len(remotefiles) > 0:
+        errors_deleting_files = 0
+        skipped_files = 0
+        for file in remotefiles:
+            swift_conn,errors_deleting_file,_,skipped_file,_ = utility.delete_object(swift_conn,swift_container,file,None,fail_tries)
+            errors_deleting_files += errors_deleting_file
+            skipped_files += skipped_file
+        print("___________")
+        print("Errors deleting files " + str(errors_deleting_files))
+        print("Skipped files to delete " + str(skipped_files))
+    else:
+        if not delete and len(remotefiles) > 0:
+            print("Delete Disabled")
+    print("___________")
+    print("___________")

swift_delete_orphan_segments.py

@@ -6,6 +6,8 @@ import authentication, utility
 
 def launch(swift_container,delete,fail_tries):
     print ("Swift container " + swift_container)
+    print ("Delete " + str(delete))
+    print ("Fail tries " + str(fail_tries))
     print("___________")
 
     swift_conn = authentication.set_authentication ()
@@ -24,8 +26,9 @@ def launch(swift_container,delete,fail_tries):
     #_____________________________________________________________________________________________________________________
     #______________________________Folders with segments
     listfoldersegmentsprefix = []
+    listfoldersegmentsprefixfull = {}
 
-    #______________________________Folders with segments : container: swift_container +"_segments"
+    #______________________________Folders with segments         container: swift_container +"_segments"
 
     #all files on container_segments -> they are only segments
     swift_conn,container_segments_objects_raw = utility.get_list(fail_tries,swift_conn,swift_container +"_segments","")
@@ -33,52 +36,87 @@ def launch(swift_container,delete,fail_tries):
     container_segments_objects = utility.list (container_segments_objects_raw,"")
     #get list only with folders path name from segments
     listfolder = []
+
     for name in container_segments_objects.keys():
+        if (swift_container +"_segments/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys():
+            listfoldersegmentsprefixfull[swift_container +"_segments/" + utility.folder_from_path(name,"/")]= []
+        listfoldersegmentsprefixfull[swift_container +"_segments/" + utility.folder_from_path(name,"/")].append(swift_container +"_segments/" + name)
+
         if utility.folder_from_path(name,"/") not in listfolder :
             listfolder.append(utility.folder_from_path(name,"/"))
     for o in listfolder:
         listfoldersegmentsprefix.append(swift_container +"_segments/" + o)
 
-    #______________________________Folders with segments : container: swift_container path: @SynologyCloudSync/
+    #______________________________Folders with segments         container: swift_container path: @SynologyCloudSync/
     #all files @SynologyCloudSync/  on container -> they are only segments
     #get dict name:size
     container_segments_objects = utility.filter_list_begin(container_objects_raw,"@SynologyCloudSync/","")
     #get list only with folders path name from segments
     listfolder = []
     for name in container_segments_objects.keys():
+        if (swift_container +"/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys():
+            listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")]=[]
+        listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")].append(swift_container +"/" + name)
         if utility.folder_from_path(name,"/") not in listfolder :
             listfolder.append(utility.folder_from_path(name,"/"))
     for o in listfolder:
         listfoldersegmentsprefix.append(swift_container +"/" + o)
 
 
-    #______________________________Folders with segments "!CB_"
+    #______________________________Folders with segments           container: swift_container path: files with !CB_
     #all files !CB_ of CloudBerry  on container -> they are only segments
     #get dict name:size
     container_segments_objects = utility.search_list(container_objects_raw,"!CB_","")
     #get list only with folders path name from segments
     listfolder = []
     for name in container_segments_objects.keys():
+        if (swift_container +"/" + utility.folder_from_path(name,"/")) not in listfoldersegmentsprefixfull.keys():
+            listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")]=[]
+        listfoldersegmentsprefixfull[swift_container +"/" + utility.folder_from_path(name,"/")].append(swift_container +"/" + name)
         if utility.folder_from_path(name,"_") not in listfolder :
             listfolder.append(utility.folder_from_path(name,"_"))
     for o in listfolder:
         listfoldersegmentsprefix.append(swift_container +"/" + o)
 
+
+    #_____________________________________________________________________________________________________________________
     '''
     for o in listfoldersegmentsprefix:
         print(o)
     '''
     #_____________________________________________________________________________________________________________________
 
+    print("___________")
     segments_to_delete = []
     segments_not_listed = []
     for manifesturlsegments in listfoldersegmentsprefix :
         if manifesturlsegments not in listfoldermanifest:
             segments_to_delete.append(manifesturlsegments)
 
+    print("Segments folders to delete: " + str(len(segments_to_delete)))
     for manifest in segments_to_delete:
         print("Segments folder to delete: " + manifest)
 
+    print("___________")
+    if delete:
+        errors_deleting_files = 0
+        skipped_files = 0
+        for short,longlist in listfoldersegmentsprefixfull.items():
+            if short in segments_to_delete:
+                for long in longlist:
+                    del_container = long.split("/")[0]
+                    del_object =  long.replace(del_container + "/","")
+                    swift_conn,errors_deleting_file,_,skipped_file,_ = utility.delete_object(swift_conn,del_container,del_object,None,fail_tries)
+                    errors_deleting_files += errors_deleting_file
+                    skipped_files += skipped_file
+        print("___________")
+        print("Errors deleting files " + str(errors_deleting_files))
+        print("Skipped files to delete " + str(skipped_files))
+    else:
+        print("Delete Disabled")
+
+    print("___________")
+    #_____
     for manifesturl in listfoldermanifest :
         if manifesturl not in listfoldersegmentsprefix:
             segments_not_listed.append(manifesturl)
@@ -86,4 +124,9 @@ def launch(swift_container,delete,fail_tries):
     for manifest in segments_not_listed:
         print("Segments not present in given segments list : " + manifest)
 
+    #_____
+
+
+
+
     swift_conn.close()

utility.py

@@ -234,6 +234,10 @@ def set_dash():
 
 
 def delete_object (swift_conn,swift_container,object,manifest,fail_tries):
+    errors_deleting_file = 0
+    errors_deleting_files_of_large_obj = 0
+    skipped_file = 0
+    skipped_files_of_large_obj = 0
     for fail_tries_counter in range (fail_tries) :
         try:
             print("Deleting " + swift_container + "/" + object)
@@ -254,9 +258,11 @@ def delete_object (swift_conn,swift_container,object,manifest,fail_tries):
                     except Exception as e:
                         print("Exception during deletion of manifest files")
                         print(e)
+                        errors_deleting_files_of_large_obj +=1
                         time.sleep(1)
                         if fail_tries_counter1 == fail_tries - 1 :
                             print("Maximum tries reached. Can't delete " + swift_container + "/" + object +" manifest files.Skipping")
+                            skipped_files_of_large_obj += 1
                         else:
                             swift_conn = authentication.set_authentication ()
                     else :
@@ -264,12 +270,26 @@ def delete_object (swift_conn,swift_container,object,manifest,fail_tries):
         except Exception as e:
             print("Exception during deletion of file")
             print(e)
+            errors_deleting_file += 1
             time.sleep(1)
             if fail_tries_counter == fail_tries - 1 :
                 print("Maximum tries reached. Can't delete " + swift_container + "/" + object +".Skipping")
+                skipped_file +=1
             else:
                 swift_conn = authentication.set_authentication ()
         else :
             break
-    return swift_conn
+    return [swift_conn,errors_deleting_file,errors_deleting_files_of_large_obj,skipped_file,skipped_files_of_large_obj]
+
+
+def list_compute_0_byte_folders (objects):
+
+    remotefiles = []
+
+    for o in objects :
+        if o["content_type"] == "application/directory" and int(o["bytes"]) == 0:
+            remotefiles.append(o["name"])
+
+    return remotefiles
+
 

utility_aes.py

@@ -1,6 +1,7 @@
 import hashlib,os
 from utility import read_in_chunks
 
+
 try:
     pycrypto = True
     from Crypto.Cipher import AES