Merge pull request #57 from dadosfera/beta

Beta

.github/workflows/deploy.yml

@@ -24,7 +24,9 @@ jobs:
         id: extract_environment
 
   deploy:
-    runs-on: self-hosted
+    needs: extract_environment
+    runs-on:
+      [self-hosted, '${{ needs.extract_environment.outputs.environment }}']
     steps:
       - name: Checkout
         uses: actions/checkout@v2

README.md

@@ -1,4 +1,4 @@
 <p align="center">
   <image src="./assets/maestro.svg" style="width:10rem">
-  <h1 align="center">Maestro</h1>
+  <h1 align="center">Maestro</h1> 
 </p>

package.json

@@ -30,7 +30,7 @@
     "@nestjs/microservices": "^8.4.3",
     "@nestjs/platform-express": "^8.4.3",
     "@nestjs/swagger": "^5.2.1",
-    "@victorradael/protospack": "^1.3.9",
+    "@victorradael/protospack": "^1.4.2",
     "axios": "^0.25.0",
     "dotenv": "^14.2.0",
     "grpc": "^1.24.11",

src/app.module.ts

@@ -29,6 +29,8 @@ import { TransformationsClientConfiguration } from './clients/transformations/cl
 import { AuthClient } from './clients/auth/client.config';
 import { InputsClientConfiguration } from './clients/inputs/client.config';
 import { PipelinesClientConfiguration } from './clients/pipelines/client.config';
+import { CatalogController } from './modules/catalog/catalog.controller';
+import { CatalogService } from './modules/catalog/catalog.service';
 
 const authClient = new AuthClient();
 const inputClient = new InputsClientConfiguration();
@@ -44,6 +46,7 @@ const transformationClient = new TransformationsClientConfiguration();
     PipelinesController,
     AuthController,
     HealthController,
+    CatalogController,
   ],
   providers: [
     InputsService,
@@ -57,6 +60,7 @@ const transformationClient = new TransformationsClientConfiguration();
     OutputsClientService,
     PipelinesClientService,
     AuthClientService,
+    CatalogService,
   ],
   imports: [
     ConfigModule.forRoot({
@@ -97,6 +101,7 @@ export class AppModule implements NestModule {
         TransformationsController,
         OutputsController,
         PipelinesController,
+        CatalogController,
       );
   }
 }

src/clients/pipelines/client.service.ts

@@ -202,4 +202,30 @@ export class PipelinesClientService implements OnModuleInit {
 
     return statusPipelineResponse;
   }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    console.log('PipelinesClientService', 'RunPipeline');
+    const statusPipelineResponse = await new Promise((resolve, reject) => {
+      this.pipelineService
+        .triggerPipeline(objectSnakeToCamel({ id, info }))
+        .subscribe({
+          next(x) {
+            resolve(x);
+          },
+          error(err) {
+            console.log('Observable Error');
+            reject(err);
+          },
+          complete() {
+            console.log('done');
+          },
+        });
+    })
+      .then((res) => res)
+      .catch((err) => {
+        throw new Error(err);
+      });
+
+    return statusPipelineResponse;
+  }
 }

src/middlewares/authentication.ts

@@ -14,7 +14,6 @@ import { HttpExceptionFilter } from 'src/error/http-exception.filter';
 @UseFilters(new HttpExceptionFilter())
 export class LoggerMiddleware implements NestMiddleware {
   use = async (request: Request, response: Response, next: NextFunction) => {
-    console.log('Middle', new Date());
     const idToken = request.get('Dadosfera-User');
     const accessToken = request.get('Authorization');
     const privateKey = process.env.JWT_PRIVATE_KEY;
@@ -31,6 +30,7 @@ export class LoggerMiddleware implements NestMiddleware {
 
     const permissions = jwtDecoded.user.permissions;
     const clienId = jwtDecoded.user.customerId;
+    const customer = jwtDecoded.user.customer;
     const userId = jwtDecoded.user.id;
 
     await verifyToken(accessToken);
@@ -50,6 +50,7 @@ export class LoggerMiddleware implements NestMiddleware {
       request.body.info = {
         customer_id: clienId,
         user_id: userId,
+        customer,
       };
 
       next();

src/modules/auth/auth.controller.ts

@@ -1,4 +1,4 @@
-import { Body, Controller, Post } from '@nestjs/common';
+import { Body, Controller, Post, UnauthorizedException } from '@nestjs/common';
 
 import { AuthClientService } from 'src/clients/auth/client.service';
 
@@ -18,7 +18,9 @@ export class AuthController {
     const tokens = await this.authClient
       .signIn({ username, password })
       .then((result) => result)
-      .catch((err) => console.log(err));
+      .catch((err) => {
+        throw new UnauthorizedException(err.message);
+      });
 
     return tokens;
   }

src/modules/catalog/catalog.controller.ts

@@ -0,0 +1,267 @@
+import { Body, Controller, Delete, Get, Param, Post } from '@nestjs/common';
+import { CatalogService } from './catalog.service';
+
+@Controller('catalog')
+export class CatalogController {
+  constructor(private catalogService: CatalogService) {}
+
+  @Get('all')
+  async catalogAll(@Body() body) {
+    console.log(`/catalog`, 'ON CATALOG ALL ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.catalogAll(body);
+
+    return res;
+  }
+
+  @Get('data_apps')
+  async dataAppsAll(@Body() body) {
+    console.log(`/catalog`, 'ON FIND ALL DATA APPS ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.dataAppsAll(body);
+
+    return res;
+  }
+
+  @Delete('data_apps/:id')
+  async dataAppsOne(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON FIND ONE DATA APP ROUTE');
+    const { id } = params;
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.dataAppsOne(body, id);
+
+    return res;
+  }
+
+  @Get('dashboard-metabase')
+  async getAllDashboardMetabase(@Body() body) {
+    console.log(`/catalog`, 'ON GET ALL DASHBOARDS METABASE ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getAllDashboardMetabase(body);
+
+    return res;
+  }
+
+  @Get('dashboard-metabase/:id')
+  async getOneDashboardMetabase(@Body() body, @Param() params) {
+    const { id } = params;
+    console.log(`/catalog`, 'ON GET ONE DASHBOARD METABASE ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getOneDashboardMetabase(body, id);
+
+    return res;
+  }
+
+  @Get('table-metadata')
+  async getAllTableMetadata(@Body() body) {
+    console.log(`/catalog`, 'ON GET ALL TABLES METADATA ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getAllTableMetadata(body);
+
+    return res;
+  }
+
+  @Delete('table-metadata')
+  async deleteOneTableMetadata(@Param() params, @Body() body) {
+    const { id } = params;
+    console.log(`/catalog`, 'ON DELETE ONE TABLE METADATA ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.deleteOneTableMetadata(body, id);
+
+    return res;
+  }
+
+  @Get('column-metadata')
+  async getOneColumnMetadata(@Body() body) {
+    console.log(`/catalog`, 'ON GET ONE COLUMN METADATA ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getOneColumnMetadata(body);
+
+    return res;
+  }
+
+  @Delete('column-metadata')
+  async deleteOneColumnMetadata(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON DELETE ONE COLUMN METADATA ROUTE');
+    const { id } = params;
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.deleteOneColumnMetadata(body, id);
+
+    return res;
+  }
+
+  @Get('data-preview')
+  async getOneDataPreview(@Body() body) {
+    console.log(`/catalog`, 'ON GET ONE DATAPREVIEW ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getOneDataPreview(body);
+
+    return res;
+  }
+
+  @Get('data-status')
+  async getDataStatus(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA STATUS ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getDataStatus(body);
+
+    return res;
+  }
+
+  @Get('data-description')
+  async getDataDescription(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA DESCRIPTION ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getDataDescription(body);
+
+    return res;
+  }
+
+  @Get('data-docs')
+  async getDataDocs(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA DOCS ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getDataDocs(body);
+
+    return res;
+  }
+
+  @Get('data-rating')
+  async getDataRating(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA RATING ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getDataRating(body);
+
+    return res;
+  }
+
+  @Get('summary-rating/:id')
+  async getSummaryRating(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON GET SUMMARY RATING ROUTE');
+    const { id } = params;
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getSummaryRating(id, body);
+
+    return res;
+  }
+
+  @Get('data-comment')
+  async getDataComment(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA COMMENT ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getDataComment(body);
+
+    return res;
+  }
+
+  @Get('data-review')
+  async getDataReview(@Body() body) {
+    console.log(`/catalog`, 'ON GET DATA REVIEW ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getDataReview(body);
+
+    return res;
+  }
+
+  @Post('tags')
+  async createTags(@Body() body) {
+    console.log(`/catalog`, 'ON CREATE TAG ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.createTags(body);
+
+    return res;
+  }
+
+  @Delete('tags/:id')
+  async deleteTags(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON DELETE TAG ROUTE');
+    const { id } = params;
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.deleteTags(body, id);
+
+    return res;
+  }
+
+  @Post('table-tags')
+  async createTableTags(@Body() body) {
+    console.log(`/catalog`, 'ON CREATE TABLE TAG ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.createTableTags(body);
+
+    return res;
+  }
+
+  @Delete('table-tags')
+  async deleteTableTags(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON CREATE TABLE TAGS ROUTE');
+    const { id } = params;
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.deleteTableTags(body, id);
+
+    return res;
+  }
+
+  @Get('table-rules')
+  async getTableRules(@Body() body) {
+    console.log(`/catalog`, 'ON GET TABLE RULES ROUTE');
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.getTableRules(body);
+
+    return res;
+  }
+
+  @Delete('table-rules')
+  async deleteTableRules(@Param() params, @Body() body) {
+    console.log(`/catalog`, 'ON DELETE TABLE RULES ROUTE');
+    const { id } = params;
+
+    const catalogService = new CatalogService();
+
+    const res = await catalogService.deleteTableRules(body, id);
+
+    return res;
+  }
+}

src/modules/catalog/catalog.service.ts

@@ -0,0 +1,282 @@
+import axios from 'axios';
+
+class CatalogService {
+  async catalogAll(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/all/`,
+    );
+    return data;
+  }
+
+  async dataAppsAll(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data_apps/`,
+    );
+    return data;
+  }
+
+  async dataAppsOne(id, body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.delete(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data_apps/${id}/`,
+    );
+    return data;
+  }
+
+  async getAllDashboardMetabase(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/dashboard-metabase/`,
+    );
+    return data;
+  }
+
+  async getOneDashboardMetabase(body, id) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/dashboard-metabase/${id}`,
+    );
+    return data;
+  }
+
+  async getAllTableMetadata(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-metadata/`,
+    );
+    return data;
+  }
+
+  async deleteOneTableMetadata(id, body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.delete(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-metadata/${id}`,
+    );
+    return data;
+  }
+
+  async getOneColumnMetadata(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/column-metadata/`,
+    );
+    return data;
+  }
+
+  async deleteOneColumnMetadata(id, body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.delete(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/column-metadata/${id}/`,
+    );
+    return data;
+  }
+
+  async getOneDataPreview(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-preview/`,
+    );
+    return data;
+  }
+
+  async getDataStatus(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-status/`,
+    );
+    return data;
+  }
+
+  async getDataDescription(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-description/`,
+    );
+    return data;
+  }
+
+  async getDataDocs(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-docs/`,
+    );
+    return data;
+  }
+
+  async getDataRating(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-rating/`,
+    );
+    return data;
+  }
+
+  async getSummaryRating(id, body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/summary-rating/${id}`,
+    );
+    return data;
+  }
+
+  async getDataComment(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-comment/`,
+    );
+    return data;
+  }
+
+  async getDataReview(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/data-review/`,
+    );
+    return data;
+  }
+
+  async createTags(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.post(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/tags/`,
+      body,
+    );
+    return data;
+  }
+
+  async deleteTags(id, body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.delete(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/tags/${id}`,
+    );
+    return data;
+  }
+
+  async createTableTags(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.post(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-tags/`,
+      body,
+    );
+    return data;
+  }
+
+  async deleteTableTags(id, body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.delete(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-tags/${id}`,
+    );
+    return data;
+  }
+
+  async getTableRules(body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.get(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-rules/`,
+      body,
+    );
+    return data;
+  }
+
+  async deleteTableRules(id, body) {
+    const customer =
+      body.info.customer.toLowerCase() === 'dadosfera'
+        ? ``
+        : `-${body.info.customer.toLowerCase()}`;
+
+    const { data } = await axios.delete(
+      `${process.env.NIMBUS_BASE_URL}${customer}.${process.env.ENV}.dadosfera/api/catalog/table-rules/${id}`,
+    );
+    return data;
+  }
+}
+
+export { CatalogService };

src/modules/pipelines/pipelines.controller.ts

@@ -14,6 +14,22 @@ import { PipelinesService } from './pipelines.service';
 export class PipelinesController {
   constructor(private pipelinesClientService: PipelinesClientService) {}
 
+  @Get('start/:id')
+  async activate(@Param() params, @Body() body) {
+    const { id } = params;
+    const { info } = body;
+
+    console.log(
+      process.env.DEV_URL + `/transformation/start/${id}`,
+      'ON START PIPELINE ROUTE',
+    );
+
+    const pipelineService = new PipelinesService(this.pipelinesClientService);
+    const response = await pipelineService.runPipeline({ id, info });
+
+    return response;
+  }
+
   @Get(':id/status')
   async getPipelineStatus(@Param() params) {
     const { id } = params;

src/modules/pipelines/pipelines.service.ts

@@ -84,4 +84,17 @@ export class PipelinesService {
       throw new HttpException(err.message, HttpStatus.NOT_FOUND);
     }
   }
+
+  async runPipeline({ id, info }: IIdRequest) {
+    try {
+      const triggerPipelineResponse = await this.pipelineClient.runPipeline({
+        id,
+        info,
+      });
+
+      return objectCamelToSnake(triggerPipelineResponse);
+    } catch (err) {
+      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
+    }
+  }
 }

src/modules/transformations/transformations.service.ts

@@ -11,15 +11,20 @@ export class TransformationsService {
   constructor(private transformationClient: TransformationsClientService) {}
 
   async create(createTransformationDto: ICreateTransformationsRequest) {
+    const { transformations, info } = createTransformationDto;
+
+    if (!transformations || transformations.length == 0) {
+      throw new HttpException('Missing transformations field.', 400);
+    }
+
     try {
-      const convertedDTO = createTransformationDto.transformations.map((tr) => {
+      const convertedDTO = transformations.map((tr) => {
         return trSnakeToCamel(tr);
       });
-
       const createTransformationResponse =
         await this.transformationClient.create({
           transformations: convertedDTO,
-          info: createTransformationDto.info,
+          info: info,
         });
 
       return createTransformationResponse;