Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
Data Flow
Data Ingestion
Ingestion DAGs
Commits
4c1c04a6
Commit
4c1c04a6
authored
Feb 17, 2021
by
Yan Sushchynski (EPAM)
Browse files
GONRG-1869: Hot fixes for FileHandler and GoogleCloudStorageClient
parent
cb859e02
Pipeline
#27016
passed with stages
in 11 minutes and 8 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
src/dags/libs/handle_file.py
View file @
4c1c04a6
...
...
@@ -75,12 +75,12 @@ class FileHandler(HeadersMixin):
self
.
token_refresher
=
token_refresher
self
.
_blob_storage_client
=
blob_storage_client
or
blob_storage
.
get_client
()
def
_get_file_from_preload_path
(
self
,
preload_path
:
str
,
def
_get_file_from_preload_path
(
self
,
preload_
file_
path
:
str
,
file
:
FileLikeObject
)
->
Tuple
[
FileLikeObject
,
str
]:
"""Get file from a preloaded path.
:param preload_path: Full URI of the file to obtain
:type preload_path: str
:param preload_
file_
path: Full URI of the file to obtain
:type preload_
file_
path: str
:return: Raw file data and content-type
:rtype: Tuple[FileLikeObject, str]
"""
...
...
src/dags/providers/gcp/gcp_blob_storage_client.py
View file @
4c1c04a6
...
...
@@ -104,7 +104,7 @@ class GoogleCloudStorageClient(BlobStorageClient):
bucket
=
self
.
_storage_client
.
bucket
(
bucket_name
)
blob
=
bucket
.
get_blob
(
source_blob_name
)
file_as_bytes
=
blob
.
download_as_
bytes
()
file_as_bytes
=
blob
.
download_as_
string
()
logger
.
debug
(
f
"File
{
source_blob_name
}
got from bucket
{
bucket_name
}
."
)
return
file_as_bytes
,
blob
.
content_type
...
...
tests/providers-unit-tests/test_gcp_blob_storage_client.py
View file @
4c1c04a6
...
...
@@ -144,7 +144,7 @@ class TestGoogleCloudStorageClient:
client_mock
.
bucket
.
assert_called_with
(
bucket_name
)
bucket_mock
.
get_blob
.
assert_called_with
(
blob_name
)
blob_mock
.
download_as_
bytes
.
assert_called_with
()
blob_mock
.
download_as_
string
.
assert_called_with
()
@
pytest
.
mark
.
parametrize
(
"uri, bucket_name, blob_name, content_type"
,
[
pytest
.
param
(
"gs://bucket_test/name_test"
,
"bucket_test"
,
"name_test"
,
"text/html"
),
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment