Weiyun1025's picture
Upload folder using huggingface_hub
2abfccb verified
# distutils: language = c++
from libcpp.string cimport string
from libcpp.vector cimport vector
from s3client cimport S3Client, get_error_list
from s3client cimport init_api as _init_api
from s3client cimport shutdown_api as _shutdown_api
ERROR_LIST = get_error_list()
ERROR_MAP = {k:v.decode('utf-8') for k,v in ERROR_LIST}
def init_api(log_level):
log_level = log_level.lower()
_init_api(log_level.encode('utf-8'))
def shutdown_api():
_shutdown_api()
class S3Error(Exception):
def __init__(self, error_name, error_message):
self.error_name = error_name
self.error_message = error_message
cdef class PyS3Client:
cdef S3Client *client
def __cinit__(self, string ak, string sk, string endpoint, bint verify_ssl, bint enable_https, bint use_dual_stack, int threads_num):
self.client = new S3Client(ak, sk, endpoint, verify_ssl, enable_https, use_dual_stack, threads_num)
def __dealloc__(self):
del self.client
def get_object(self, string bucket, string key, string range):
cdef string error_message, result
cdef int error_type
ret = self.client.get_object(bucket, key, error_type, error_message, result, range)
if ret == 0:
return result
else:
error_name = ERROR_MAP.get(error_type, 'Undefined')
raise S3Error(error_name, error_message)
def multipart_download_concurrency(self, string bucket, string key, string filename):
cdef string error_message, result
cdef int error_type
ret = self.client.multipart_download_concurrency(bucket, key, filename, error_type, error_message)
if ret == 0:
return ret
else:
error_name = ERROR_MAP.get(error_type, 'Undefined')
raise S3Error(error_name, error_message)
def put_object(self, string bucket, string key, string data):
cdef string error_message, result
cdef int error_type
ret = self.client.put_object(bucket, key, data, error_type, error_message)
if ret == 0:
return data.size()
else:
error_name = ERROR_MAP.get(error_type, 'Undefined')
raise S3Error(error_name, error_message)
def multipart_upload_concurrency(self, string bucket, string key, string filename):
cdef string error_message, result
cdef int error_type
ret = self.client.multipart_upload_concurrency(bucket, key, filename, error_type, error_message)
if ret == 0:
return ret
else:
error_name = ERROR_MAP.get(error_type, 'Undefined')
raise S3Error(error_name, error_message)
def delete(self, string bucket, string key):
cdef string error_message, result
cdef int error_type
ret = self.client.delete_obj(bucket, key, error_type, error_message)
return ret
def contains(self, string bucket, string key):
cdef string error_message, result
cdef int error_type
ret = self.client.contains(bucket, key, error_type, error_message)
return ret
def list(self, string bucket, string key):
cdef string error_message, result
cdef int error_type
ret = self.client.list(bucket, key, error_type, error_message)
return ret