first release. readme will follow up
This commit is contained in:
222
utility.py
Normal file
222
utility.py
Normal file
@@ -0,0 +1,222 @@
|
||||
__author__ = 'xgiovio'
|
||||
|
||||
import authentication
|
||||
import sys, platform,os
|
||||
import time
|
||||
|
||||
|
||||
def get_local_files (localpath):
|
||||
print("Building local files list")
|
||||
localfiles = {}
|
||||
for root, dirs, files in os.walk(localpath, topdown=True):
|
||||
for name in files:
|
||||
localfiles[os.path.join(root.replace(localpath,""), name)]=int(os.stat(os.path.join(root, name)).st_size)
|
||||
|
||||
#skip dirs
|
||||
'''
|
||||
for name in dirs:
|
||||
print(os.path.join(root, name))
|
||||
'''
|
||||
#end skip
|
||||
return localfiles
|
||||
|
||||
def get_list (fail_tries,swift_conn,swift_container,prefix) :
|
||||
for fail_tries_counter in range (fail_tries) :
|
||||
try:
|
||||
print ("Downloading remote list for " + swift_container + " with prefix " +prefix + " ... ")
|
||||
headers,objects = swift_conn.get_container(swift_container, prefix =prefix,full_listing=True )
|
||||
except Exception as e:
|
||||
print("Exception during the download of remote list")
|
||||
print(e)
|
||||
time.sleep(1)
|
||||
if fail_tries_counter == fail_tries - 1 :
|
||||
print("Maximum tries reached. Can't download remote list for container " + swift_container + " with prefix " +prefix +". Exiting.")
|
||||
sys.exit(-1)
|
||||
else:
|
||||
swift_conn = authentication.set_authentication ()
|
||||
else :
|
||||
break
|
||||
return [swift_conn,objects]
|
||||
|
||||
def filter_list_begin(objects,beginpattern,prefix):
|
||||
filtered = {}
|
||||
for o in objects :
|
||||
if o["content_type"] != "application/directory":
|
||||
if (o["name"]).find(beginpattern)== 0:
|
||||
filtered[(o["name"].replace(prefix,""))]=int(o["bytes"])
|
||||
return filtered
|
||||
|
||||
def search_list (objects,pattern,prefix):
|
||||
filtered = {}
|
||||
for o in objects :
|
||||
if o["content_type"] != "application/directory":
|
||||
if pattern in o["name"]:
|
||||
filtered[(o["name"].replace(prefix,""))]=int(o["bytes"])
|
||||
return filtered
|
||||
|
||||
|
||||
def list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix):
|
||||
|
||||
remotefiles = {}
|
||||
remotefiles_md5 = {}
|
||||
byte0real = 0
|
||||
byte0manifest = 0
|
||||
|
||||
for o in objects :
|
||||
if o["content_type"] != "application/directory":
|
||||
if int(o["bytes"]) == 0 :
|
||||
print ("Requesting metadata for 0byte file " + o["name"] )
|
||||
for fail_tries_counter in range (fail_tries) :
|
||||
try:
|
||||
oheaders = swift_conn.head_object(swift_container,o["name"])
|
||||
except Exception as e:
|
||||
print("Exception during the request of metadata")
|
||||
print(e)
|
||||
time.sleep(1)
|
||||
if fail_tries_counter == fail_tries - 1 :
|
||||
print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.")
|
||||
sys.exit(-1)
|
||||
else:
|
||||
swift_conn = authentication.set_authentication ()
|
||||
else :
|
||||
break
|
||||
if "x-object-manifest" in oheaders.keys():
|
||||
print ("0byte file " + o["name"] + " e' un large file" )
|
||||
o["bytes"] = oheaders["content-length"]
|
||||
if "_xg10v10_" in oheaders["x-object-manifest"]:
|
||||
split1 = oheaders["x-object-manifest"].split("_xg10v10_")[0]
|
||||
split2 = split1.split("/")[1]
|
||||
o["hash"] = split2
|
||||
else:
|
||||
print("Impossible to get remote large file md5. Cause: Not uploaded with xgiovio method (md5 in x-object-manifest)")
|
||||
o["hash"] = "0"
|
||||
byte0manifest = byte0manifest + 1
|
||||
else:
|
||||
print ("0byte file " + o["name"] + " e' un file normale" )
|
||||
byte0real = byte0real + 1
|
||||
remotefiles[remote_dash_replace(o["name"].replace(prefix,""))]=int(o["bytes"])
|
||||
remotefiles_md5[remote_dash_replace(o["name"].replace(prefix,""))]=o["hash"]
|
||||
|
||||
|
||||
return [byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5]
|
||||
|
||||
|
||||
|
||||
def list_compute_correct_names_for_enctyption (objects,prefix):
|
||||
|
||||
remotefiles = {}
|
||||
|
||||
for o in objects :
|
||||
if o["content_type"] != "application/directory":
|
||||
full = remote_dash_replace(o["name"].replace(prefix,""))
|
||||
if (full.endswith("_xg10v10_encrypted")):
|
||||
encrypted_name_only = full.split("_xg10v10_")[0] + "_xg10v10_encrypted"
|
||||
else:
|
||||
encrypted_name_only = full
|
||||
remotefiles[encrypted_name_only] = full
|
||||
|
||||
return remotefiles
|
||||
|
||||
|
||||
def list_compute_manifest (fail_tries,objects,swift_conn,swift_container,prefix):
|
||||
|
||||
remotefiles = {}
|
||||
byte0manifest = 0
|
||||
|
||||
for o in objects :
|
||||
if o["content_type"] != "application/directory":
|
||||
if int(o["bytes"]) == 0 :
|
||||
print ("Requesting metadata for 0byte file " + o["name"] )
|
||||
for fail_tries_counter in range (fail_tries) :
|
||||
try:
|
||||
oheaders = swift_conn.head_object(swift_container,o["name"])
|
||||
except Exception as e:
|
||||
print("Exception during the request of metadata")
|
||||
print(e)
|
||||
time.sleep(1)
|
||||
if fail_tries_counter == fail_tries - 1 :
|
||||
print("Maximum tries reached. Can't download sizes for all large files on container " + swift_container + " with prefix: " +prefix +". Exiting.")
|
||||
sys.exit(-1)
|
||||
else:
|
||||
swift_conn = authentication.set_authentication ()
|
||||
else :
|
||||
break
|
||||
if "x-object-manifest" in oheaders.keys():
|
||||
byte0manifest = byte0manifest + 1
|
||||
remotefiles[(o["name"].replace(prefix,""))]=oheaders["x-object-manifest"]
|
||||
|
||||
return [byte0manifest,swift_conn,remotefiles]
|
||||
|
||||
def list (objects,prefix):
|
||||
|
||||
remotefiles = {}
|
||||
for o in objects :
|
||||
if o["content_type"] != "application/directory":
|
||||
remotefiles[(o["name"].replace(prefix,""))]=int(o["bytes"])
|
||||
|
||||
return remotefiles
|
||||
|
||||
def listall (objects,prefix):
|
||||
|
||||
remotefiles = {}
|
||||
for o in objects :
|
||||
remotefiles[(o["name"].replace(prefix,""))]=int(o["bytes"])
|
||||
return remotefiles
|
||||
|
||||
def check_segments_size (size_limit_reading_os,size_limit_to_segment):
|
||||
if size_limit_reading_os%2 != 0 or size_limit_to_segment%2 != 0 or size_limit_reading_os > size_limit_to_segment or size_limit_reading_os < 16:
|
||||
print ("Error size segments")
|
||||
sys.exit(-2)
|
||||
|
||||
def check_segments_size_single (size_limit_reading_os):
|
||||
if size_limit_reading_os%2 != 0 or size_limit_reading_os < 16:
|
||||
print ("Error size segments")
|
||||
sys.exit(-2)
|
||||
|
||||
def file_only_name (stringa,delimiter):
|
||||
return stringa.split(delimiter)[len(stringa.split(delimiter))-1]
|
||||
|
||||
def folder_from_path (stringa,delimiter):
|
||||
string_to_ret =""
|
||||
for i in range(len(stringa.split(delimiter)) - 1):
|
||||
string_to_ret = string_to_ret + stringa.split(delimiter)[i] + delimiter
|
||||
return string_to_ret
|
||||
|
||||
def check_end_slash (stringa) :
|
||||
lastchar = stringa[len(stringa) - 1]
|
||||
if lastchar == "\\" or lastchar == "/" :
|
||||
return True
|
||||
return False
|
||||
def check_start_slash (stringa) :
|
||||
firstchar = stringa[0]
|
||||
if firstchar == "\\" or firstchar == "/" :
|
||||
return True
|
||||
return False
|
||||
|
||||
def read_in_chunks(file_object, chunk_size):
|
||||
while True:
|
||||
data = file_object.read(chunk_size)
|
||||
if not data:
|
||||
break
|
||||
yield data
|
||||
|
||||
def total_size_encrypted (original_size) :
|
||||
if original_size % 16 == 0:
|
||||
return original_size + 17
|
||||
return original_size + ( 16 - original_size % 16) + 17
|
||||
|
||||
|
||||
def dash_replace (string):
|
||||
if platform.system() == "Windows":
|
||||
return string.replace("\\","/")
|
||||
return string
|
||||
|
||||
def remote_dash_replace (string):
|
||||
if platform.system() == "Windows":
|
||||
return string.replace("/","\\")
|
||||
return string
|
||||
|
||||
def set_dash():
|
||||
if platform.system() == "Windows":
|
||||
return "\\"
|
||||
return "/"
|
||||
Reference in New Issue
Block a user