Commit 8f1b056e authored by Diego Molteni's avatar Diego Molteni
Browse files

Merge branch 'slb/dm3/hpc' into 'master'

Fixed async methods, Updated the locker, Reviewed the catalogue access model

See merge request !57
parents fd871295 ea62a27a
Pipeline #34720 passed with stages
in 6 minutes and 24 seconds
......@@ -43,12 +43,12 @@ npm run code-coverage
--run-parallel
# run the linter on sources
tslint -c tslint.json 'src/**/*.ts'
tslint -c tslint.json -p tsconfig.json 'src/**/*.ts'
```
## Environment configuration
Environment variables can be provided with a `.env` file in the root of the project to be consumed
by [dotenv](https://github.com/motdotla/dotenv). Environment variables are [preloaded](https://github.com/motdotla/dotenv#preload)
by the `npm start` command with the argument `-r dotenv/config`. A template `.env` file can be found
by the `npm start` command with the argument `-r dotenv/config`. A template `.env` file can be found
in `/docs/templates/.env-sample`.
......@@ -41,7 +41,7 @@ export class AWSConfig extends Config {
// Logger
AWSConfig.LOGGER_LEVEL = process.env.LOGGER_LEVEL || 'info';
Config.initServiceConfiguration({
await Config.initServiceConfiguration({
SERVICE_ENV: process.env.SERVICE_ENV,
SERVICE_PORT: +process.env.PORT || 5000,
API_BASE_PATH: process.env.API_BASE_PATH,
......
......@@ -125,7 +125,7 @@ export class AWSStorage extends AbstractStorage {
Body: data
};
try {
this.s3.putObject(params).promise();
await this.s3.putObject(params).promise();
} catch (err) {
// tslint:disable-next-line:no-console
console.log(err.code + ': ' + err.message);
......@@ -140,7 +140,7 @@ export class AWSStorage extends AbstractStorage {
Key: folder + '/' + objectName
};
try {
this.s3.deleteObject(params).promise();
await this.s3.deleteObject(params).promise();
} catch (err) {
// tslint:disable-next-line:no-console
console.log(err.code + ': ' + err.message);
......
......@@ -96,6 +96,7 @@ export class GCS extends AbstractStorage {
if (async) {
await this.getStorageclient().bucket(bucketName).deleteFiles({ prefix, force: true });
} else {
// tslint:disable-next-line: no-floating-promises
this.getStorageclient().bucket(bucketName).deleteFiles({ prefix, force: true });
}
}
......
......@@ -33,6 +33,7 @@ export class StorageJobManager {
})
// setup job processing callback
// tslint:disable-next-line: no-floating-promises
StorageJobManager.copyJobsQueue.process(50, (input) => {
return StorageJobManager.copy(input)
})
......@@ -110,11 +111,17 @@ export class StorageJobManager {
registeredDataset.transfer_status = TransferStatus.Completed
DatasetDAO.update(journalClient, registeredDataset, registeredDatasetKey)
await DatasetDAO.update(journalClient, registeredDataset, registeredDatasetKey)
await Locker.releaseMutex(cacheMutex, datasetToPath)
await Locker.unlock(journalClient, input.data.datasetTo, input.data.datasetTo.sbit);
await Locker.unlock(journalClient, input.data.datasetFrom, input.data.readlockId);
const lockKeyFrom = input.data.datasetFrom.tenant + '/' + input.data.datasetFrom.subproject +
input.data.datasetFrom.path + input.data.datasetFrom.name;
await Locker.unlock(lockKeyFrom, input.data.readlockId);
const lockKeyTo = input.data.datasetTo.tenant + '/' + input.data.datasetTo.subproject +
input.data.datasetTo.path + input.data.datasetTo.name;
await Locker.unlock(lockKeyTo, input.data.datasetTo.sbit);
LoggerFactory.build(Config.CLOUDPROVIDER).info(
'[copy-transfer] completed copy operations to ' + datasetToPath)
......@@ -131,7 +138,7 @@ export class StorageJobManager {
// try to update the status to aborted if possible
if (registeredDataset) {
registeredDataset.transfer_status = TransferStatus.Aborted
DatasetDAO.update(journalClient, registeredDataset, registeredDatasetKey)
await DatasetDAO.update(journalClient, registeredDataset, registeredDatasetKey)
}
throw err
......
......@@ -50,7 +50,7 @@ async function ServerStart() {
TraceFactory.build(Config.CLOUDPROVIDER).start();
}
new (await import('./server')).Server().start();
await new (await import('./server')).Server().start();
} catch (error) {
// tslint:disable-next-line
......@@ -60,4 +60,5 @@ async function ServerStart() {
}
// tslint:disable-next-line: no-floating-promises
ServerStart();
......@@ -25,7 +25,7 @@ export class DatasetDAO {
public static async register(
journalClient: IJournal | IJournalTransaction, datasetEntity: {key: object, data: DatasetModel}) {
datasetEntity.data.ctag = Utils.makeID(16);
journalClient.save(datasetEntity);
await journalClient.save(datasetEntity);
}
public static async get(
......@@ -219,7 +219,8 @@ export class DatasetDAO {
entity.tenant = entity.tenant || tenantName;
entity.ctag = entity.ctag || '0000000000000000';
entity.readonly = entity.readonly || false;
const lockres = await Locker.getLockFromModel(entity);
const lockKey = entity.tenant + '/' + entity.subproject + entity.path + entity.name;
const lockres = await Locker.getLock(lockKey);
if (!lockres) { // unlocked
entity.sbit = null;
entity.sbit_count = 0;
......
This diff is collapsed.
This diff is collapsed.
......@@ -22,80 +22,80 @@ const router = Router();
// register a new dataset
router.post('/tenant/:tenantid/subproject/:subprojectid/dataset/:datasetid',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.Register);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.Register);
});
// get a dataset
router.get('/tenant/:tenantid/subproject/:subprojectid/dataset/:datasetid',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.Get);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.Get);
});
// list all datasets in a subporject
router.get('/tenant/:tenantid/subproject/:subprojectid',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.List);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.List);
});
// delete a dataset
router.delete('/tenant/:tenantid/subproject/:subprojectid/dataset/:datasetid',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.Delete);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.Delete);
});
// patch a Dataset
router.patch('/tenant/:tenantid/subproject/:subprojectid/dataset/:datasetid',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.Patch);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.Patch);
});
// Lock a Dataset for opening
router.put('/tenant/:tenantid/subproject/:subprojectid/dataset/:datasetid/lock',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.Lock);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.Lock);
});
// UnLock a Dataset
router.put('/tenant/:tenantid/subproject/:subprojectid/dataset/:datasetid/unlock',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.UnLock);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.UnLock);
});
// check if a list of datasets exist in a subproject
router.post('/tenant/:tenantid/subproject/:subprojectid/exist',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.Exists);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.Exists);
});
// retrieve the dataset size for a list of datasets
router.post('/tenant/:tenantid/subproject/:subprojectid/sizes',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.Sizes);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.Sizes);
});
// check the permissions of a user on a dataset
router.get('/tenant/:tenantid/subproject/:subprojectid/dataset/:datasetid/permission',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.Permission);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.Permission);
});
// check the dataset ctag
router.get('/tenant/:tenantid/subproject/:subprojectid/dataset/:datasetid/ctagcheck',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.CheckCTag);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.CheckCTag);
});
// list a path content
router.get('/tenant/:tenantid/subproject/:subprojectid/readdsdirfulllist',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.ListContent);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.ListContent);
});
// insert tags for a existing dataset
router.put('/tenant/:tenantid/subproject/:subprojectid/dataset/:datasetid/gtags',
(req: expRequest, res: expResponse) => {
DatasetHandler.handler(req, res, DatasetOP.PutTags);
async (req: expRequest, res: expResponse) => {
await DatasetHandler.handler(req, res, DatasetOP.PutTags);
});
export { router as DatasetRouter };
......@@ -21,13 +21,13 @@ import { GeneralOP } from './optype';
const router = Router();
// get the service status response [jwt not required by esp]
router.get('/', (req: expRequest, res: expResponse) => {
GeneralHandler.handler(req, res, GeneralOP.Status);
router.get('/', async (req: expRequest, res: expResponse) => {
await GeneralHandler.handler(req, res, GeneralOP.Status);
});
// get the service status response [jwt required by esp]
router.get('/access', (req: expRequest, res: expResponse) => {
GeneralHandler.handler(req, res, GeneralOP.Access);
router.get('/access', async (req: expRequest, res: expResponse) => {
await GeneralHandler.handler(req, res, GeneralOP.Access);
});
export { router as GeneralRouter };
......@@ -21,18 +21,18 @@ import { ImpTokenOP } from './optype';
const router = Router();
// generate an impersonation token
router.post('/', (req: expRequest, res: expResponse) => {
ImpTokenHandler.handler(req, res, ImpTokenOP.Generate);
router.post('/', async (req: expRequest, res: expResponse) => {
await ImpTokenHandler.handler(req, res, ImpTokenOP.Generate);
});
// refresh an impersonation token
router.put('/', (req: expRequest, res: expResponse) => {
ImpTokenHandler.handler(req, res, ImpTokenOP.Refresh);
router.put('/', async (req: expRequest, res: expResponse) => {
await ImpTokenHandler.handler(req, res, ImpTokenOP.Refresh);
});
// patch an impersonation token
router.patch('/', (req: expRequest, res: expResponse) => {
ImpTokenHandler.handler(req, res, ImpTokenOP.Patch);
router.patch('/', async (req: expRequest, res: expResponse) => {
await ImpTokenHandler.handler(req, res, ImpTokenOP.Patch);
});
export { router as ImpTokenRouter };
......@@ -295,6 +295,7 @@ export class SubProjectHandler {
}
// delete the bucket resource (to perform after files deletions)
// tslint:disable-next-line: no-floating-promises (we want it async)
storage.deleteBucket(subproject.gcs_bucket);
}
......
......@@ -21,28 +21,28 @@ import { SubProjectOP } from './optype';
const router = Router();
// register a new subproject
router.post('/tenant/:tenantid/subproject/:subprojectid', (req: expRequest, res: expResponse) => {
SubProjectHandler.handler(req, res, SubProjectOP.Create);
router.post('/tenant/:tenantid/subproject/:subprojectid', async (req: expRequest, res: expResponse) => {
await SubProjectHandler.handler(req, res, SubProjectOP.Create);
});
// get a subproject
router.get('/tenant/:tenantid/subproject/:subprojectid', (req: expRequest, res: expResponse) => {
SubProjectHandler.handler(req, res, SubProjectOP.Get);
router.get('/tenant/:tenantid/subproject/:subprojectid', async (req: expRequest, res: expResponse) => {
await SubProjectHandler.handler(req, res, SubProjectOP.Get);
});
// delete a subproject
router.delete('/tenant/:tenantid/subproject/:subprojectid', (req: expRequest, res: expResponse) => {
SubProjectHandler.handler(req, res, SubProjectOP.Delete);
router.delete('/tenant/:tenantid/subproject/:subprojectid', async (req: expRequest, res: expResponse) => {
await SubProjectHandler.handler(req, res, SubProjectOP.Delete);
});
// patch a subproject
router.patch('/tenant/:tenantid/subproject/:subprojectid', (req: expRequest, res: expResponse) => {
SubProjectHandler.handler(req, res, SubProjectOP.Patch);
router.patch('/tenant/:tenantid/subproject/:subprojectid', async (req: expRequest, res: expResponse) => {
await SubProjectHandler.handler(req, res, SubProjectOP.Patch);
});
// list all subprojects in a tenant
router.get('/tenant/:tenantid/', (req: expRequest, res: expResponse) => {
SubProjectHandler.handler(req, res, SubProjectOP.List);
router.get('/tenant/:tenantid/', async (req: expRequest, res: expResponse) => {
await SubProjectHandler.handler(req, res, SubProjectOP.List);
});
export { router as SubprojectRouter };
......@@ -21,23 +21,23 @@ import { AppOp } from './optype';
const router = Router();
// register a new application
router.post('/', (req: expRequest, res: expResponse) => {
AppHandler.handler(req, res, AppOp.Register);
router.post('/', async (req: expRequest, res: expResponse) => {
await AppHandler.handler(req, res, AppOp.Register);
});
// list the registered applications
router.get('/', (req: expRequest, res: expResponse) => {
AppHandler.handler(req, res, AppOp.List);
router.get('/', async (req: expRequest, res: expResponse) => {
await AppHandler.handler(req, res, AppOp.List);
});
// register a trusted application
router.post('/trusted', (req: expRequest, res: expResponse) => {
AppHandler.handler(req, res, AppOp.RegisterTrusted);
router.post('/trusted', async (req: expRequest, res: expResponse) => {
await AppHandler.handler(req, res, AppOp.RegisterTrusted);
});
// list the truested applications
router.get('/trusted', (req: expRequest, res: expResponse) => {
AppHandler.handler(req, res, AppOp.ListTrusted);
router.get('/trusted', async (req: expRequest, res: expResponse) => {
await AppHandler.handler(req, res, AppOp.ListTrusted);
});
export { router as SvcAppRouter };
......@@ -95,7 +95,7 @@ export class TenantDAO {
default_acls: tenantName,
gcpid: tenantName} : undefined);
serviceClient.delete(serviceClient.createKey({
await serviceClient.delete(serviceClient.createKey({
namespace: Config.ORGANIZATION_NS,
path: [Config.TENANTS_KIND, tenantName],
}));
......
......@@ -20,23 +20,23 @@ import { TenantOP } from './optype';
const router = Router();
router.get('/sdpath', (req: expRequest, res: expResponse) => {
TenantHandler.handler(req, res, TenantOP.GETSDPATH);
router.get('/sdpath', async (req: expRequest, res: expResponse) => {
await TenantHandler.handler(req, res, TenantOP.GETSDPATH);
});
// create a tenant project
router.post('/:tenantid', (req: expRequest, res: expResponse) => {
TenantHandler.handler(req, res, TenantOP.CREATE);
router.post('/:tenantid', async (req: expRequest, res: expResponse) => {
await TenantHandler.handler(req, res, TenantOP.CREATE);
});
// get a tenant project
router.get('/:tenantid', (req: expRequest, res: expResponse) => {
TenantHandler.handler(req, res, TenantOP.GET);
router.get('/:tenantid', async (req: expRequest, res: expResponse) => {
await TenantHandler.handler(req, res, TenantOP.GET);
});
// delete a tenant project
router.delete('/:tenantid', (req: expRequest, res: expResponse) => {
TenantHandler.handler(req, res, TenantOP.DELETE);
router.delete('/:tenantid', async (req: expRequest, res: expResponse) => {
await TenantHandler.handler(req, res, TenantOP.DELETE);
});
export { router as TenantRouter };
......@@ -177,7 +177,7 @@ export class UserHandler {
req.headers.authorization, datagroup, userEmail,
tenant.esd, req[Config.DE_FORWARD_APPKEY]));
// add user as owner of the
// add user as owner
await AuthGroups.addUserToGroup(req.headers.authorization,
datagroup, userEmail, tenant.esd, req[Config.DE_FORWARD_APPKEY], 'OWNER');
......
......@@ -21,23 +21,23 @@ import { UserOP } from './optype';
const router = Router();
// Add a registered user to a resource
router.put('/', (req: expRequest, res: expResponse) => {
UserHandler.handler(req, res, UserOP.Add);
router.put('/', async (req: expRequest, res: expResponse) => {
await UserHandler.handler(req, res, UserOP.Add);
});
// Retrieve user roles
router.get('/', (req: expRequest, res: expResponse) => {
UserHandler.handler(req, res, UserOP.List);
router.get('/', async (req: expRequest, res: expResponse) => {
await UserHandler.handler(req, res, UserOP.List);
});
// Retrieve user roles
router.delete('/', (req: expRequest, res: expResponse) => {
UserHandler.handler(req, res, UserOP.Remove);
router.delete('/', async (req: expRequest, res: expResponse) => {
await UserHandler.handler(req, res, UserOP.Remove);
});
// retrieve the roles of the user
router.get('/roles', (req: expRequest, res: expResponse) => {
UserHandler.handler(req, res, UserOP.Roles);
router.get('/roles', async (req: expRequest, res: expResponse) => {
await UserHandler.handler(req, res, UserOP.Roles);
});
export { router as UserRouter };
......@@ -289,7 +289,8 @@ export class UtilityHandler {
try {
// check if a copy is already in progress from a previous request
const toDatasetLock = await Locker.getLockFromModel(datasetTo)
const lockKeyTo = datasetTo.tenant + '/' + datasetTo.subproject + datasetTo.path + datasetTo.name;
const toDatasetLock = await Locker.getLock(lockKeyTo)
const results = await DatasetDAO.get(journalClient, datasetTo)
preRegisteredDataset = results[0] as DatasetModel
......@@ -326,10 +327,11 @@ export class UtilityHandler {
' already exists'));
}
writeLockSession = await Locker.createWriteLock(datasetTo);
writeLockSession = await Locker.createWriteLock(lockKeyTo);
// check if the source can be opened for read (no copy on writelock dataset)
const fromDatasetLock = await Locker.getLockFromModel(datasetFrom);
const lockKeyFrom = datasetFrom.tenant + '/' + datasetFrom.subproject + datasetFrom.path + datasetFrom.name;
const fromDatasetLock = await Locker.getLock(lockKeyFrom);
if (fromDatasetLock && Locker.isWriteLock(fromDatasetLock)) {
throw (Error.make(Error.Status.BAD_REQUEST,
......@@ -341,7 +343,7 @@ export class UtilityHandler {
let readlock: { id: string, cnt: number; };
if (userInputs.lock) {
readlock = await Locker.acquireReadLock(journalClient, datasetFrom);
readlock = await Locker.acquireReadLock(lockKeyFrom);
}
if (FeatureFlags.isEnabled(Feature.LEGALTAG)) {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment