Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in / Register
Toggle navigation
Menu
Open sidebar
Open Subsurface Data Universe Software
Platform
Data Flow
Data Ingestion
osdu-ingestion-lib
Commits
17db80dc
Commit
17db80dc
authored
Jan 08, 2022
by
Vadzim Kulyba (EPAM)
Committed by
harshit aggarwal
Jan 08, 2022
Browse files
feat(azure): add file handing support for azure provider
parent
164c46a1
Changes
1
Hide whitespace changes
Inline
Side-by-side
osdu_ingestion/libs/handle_file.py
View file @
17db80dc
...
...
@@ -19,6 +19,7 @@ import dataclasses
import
io
import
json
import
logging
import
os
from
typing
import
List
,
Tuple
import
requests
...
...
@@ -26,7 +27,7 @@ import tenacity
from
osdu_api.auth.authorization
import
TokenRefresher
,
authorize
from
osdu_api.providers
import
blob_storage
from
osdu_api.providers.types
import
BlobStorageClient
,
FileLikeObject
from
osdu_api.utils.request
import
request_raiser
from
osdu_ingestion.libs.constants
import
RETRIES
,
WAIT
from
osdu_ingestion.libs.context
import
Context
from
osdu_ingestion.libs.exceptions
import
InvalidFileRecordData
...
...
@@ -123,6 +124,13 @@ class FileHandler(HeadersMixin):
unsigned_url
=
None
,
kind
=
None
)
@
tenacity
.
retry
(
**
RETRY_SETTINGS
)
@
authorize
()
def
_send_get_request
(
self
,
headers
:
dict
,
url
:
str
)
->
requests
.
Response
:
response
=
requests
.
get
(
url
,
headers
=
headers
)
logger
.
debug
(
response
)
return
response
@
tenacity
.
retry
(
**
RETRY_SETTINGS
)
@
authorize
()
def
_send_post_request
(
self
,
headers
:
dict
,
url
:
str
,
request_body
:
str
)
->
requests
.
Response
:
...
...
@@ -132,10 +140,14 @@ class FileHandler(HeadersMixin):
return
response
@
tenacity
.
retry
(
**
RETRY_SETTINGS
)
@
authorize
()
def
_send_get_request
(
self
,
headers
:
dict
,
url
:
str
)
->
requests
.
Response
:
response
=
requests
.
get
(
url
,
headers
=
headers
)
logger
.
debug
(
response
)
@
request_raiser
def
_send_put_request
(
self
,
headers
:
dict
,
url
:
str
,
request_body
:
str
)
->
requests
.
Response
:
if
os
.
getenv
(
"CLOUD_PROVIDER"
)
==
"azure"
:
headers
[
"x-ms-blob-type"
]
=
"BlockBlob"
logger
.
debug
(
f
"
{
request_body
}
"
)
response
=
requests
.
put
(
url
,
request_body
,
headers
=
headers
)
logger
.
debug
(
response
.
content
)
return
response
def
_get_upload_signed_url
(
self
,
headers
:
dict
)
->
FileUploadUrlResponse
:
...
...
@@ -187,7 +199,7 @@ class FileHandler(HeadersMixin):
"""
logger
.
debug
(
"Uploading file."
)
buffer
.
seek
(
0
)
requests
.
put
(
signed_url
,
buffer
.
read
()
,
headers
=
headers
)
self
.
_send_put_request
(
headers
,
signed_url
,
buffer
.
read
())
logger
.
debug
(
"File uploaded."
)
def
_get_file_location_request
(
self
,
headers
:
dict
,
file_id
:
str
)
->
str
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment