filesystem
Cloud functions
filesystem.utils
General utils functions
upload_and_delete_local_files
def upload_and_delete_local_files(dst_path: str, local_path: Path)
Upload an entire folder and delete local files to either AWS or GCP
get_maximum_batch_id
def get_maximum_batch_id(path: str)
Get the last batch ID. Works with GCS, AWS, and local.
Arguments:
-
path
- the path folder to look in. Begin with 'gs://' for GCS. Begin with 's3://' for AWS S3. -
Returns
- the maximum batch id of data inpath
.
delete_all_files_in_temp_path
def delete_all_files_in_temp_path(path: Union[Path, str], delete_dirs: bool = False)
Delete all the files in a temporary path. Option to delete the folders or not
check_path_exists
def check_path_exists(path: Union[str, Path])
Raises a RuntimeError if path
does not exist in the local filesystem.
rename_file
def rename_file(remote_file: str, new_filename: str)
Rename file within one filesystem
Arguments:
remote_file
- The current file namenew_filename
- What the file should be renamed too
get_all_filenames_in_path
def get_all_filenames_in_path(path: Union[str, Path]) -> List[str]
Get all the files names from one folder in gcp
Arguments:
-
path
- the path that we should look in -
Returns
- a list of files names represented as strings.
download_to_local
def download_to_local(remote_filename: str, local_filename: str)
Download file from gcs.
Arguments:
remote_filename
- the file name, should start with gs:// or s3://local_filename
- the local filename
upload_one_file
def upload_one_file(remote_filename: str, local_filename: str)
Upload one file to aws or gcp
Arguments:
remote_filename
- the aws/gcp key namelocal_filename
- the local file name
make_folder
def make_folder(path: Union[str, Path])
Make folder