Merge pull request #163 from dadosfera/new-ecr-image

FIX: new ecr image

.github/workflows/deploy-manually.yml

@@ -113,10 +113,12 @@ jobs:
           ENV: ${{ needs.extract_environment.outputs.environment }}
           IMAGE_TAG: ${{ needs.semantic_release.outputs.new_release_version }}
           ACCOUNT_ID: ${{ steps.aws.outputs.aws-account-id }}
+          NODE_EXPORTER_URL: ${{needs.extract_environment.outputs.environment == 'prd' && '611330257153.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest' || '468720548566.dkr.ecr.us-east-1.amazonaws.com\/monitoring\/node_exporter:latest'}}
         run: |
           sed -i -e "s/\${ENV}/$ENV/g" docker-compose.yml
           sed -i -e "s/\${IMAGE_TAG}/$IMAGE_TAG/g" docker-compose.yml
           sed -i -e "s/\${ACCOUNT_ID}/$ACCOUNT_ID/g" docker-compose.yml
+          sed -i -e "s/\${NODE_EXPORTER_URL}/$NODE_EXPORTER_URL/g" docker-compose.yml
           zip deploy.zip docker-compose.yml  -r .ebextensions
 
       - name: Deploy AWS Beanstalk

docker-compose.yml

@@ -10,7 +10,7 @@ services:
       - .env
 
   node_exporter:
-    image: prom/node-exporter:v1.3.1
+    image: ${NODE_EXPORTER_URL}
     container_name: node_exporter
     restart: always
     volumes:

src/app.module.ts

@@ -46,6 +46,7 @@ import { ProductboardModule } from './modules/productboard/productboard.module';
     NetworkConfigModule,
     ConnectionTestModule,
     PipelinesModule,
+    PipelinesV2Module,
     TransformationsModule,
     HealthModule,
     CatalogModule,
@@ -53,7 +54,6 @@ import { ProductboardModule } from './modules/productboard/productboard.module';
     RolesModule,
     InputsModule,
     OauthModule,
-    PipelinesV2Module,
     CatalogModule,
     ProductboardModule,
   ],

src/authentication/permissions.enum.ts

@@ -128,7 +128,7 @@ export const PERMISSIONS_GROUPS = {
       DELETE: {
         seqid: 38,
         claim: 'connection:delete',
-        usage: PermissionUsages.INTERNAL,
+        usage: PermissionUsages.PUBLIC,
         name: {
           'pt-br': 'Excluir Fonte',
           'en-us': 'Remove Source',

src/error/grpc-to-http-exception.filter.ts

@@ -3,7 +3,6 @@ import {
   Catch,
   ArgumentsHost,
   HttpException,
-  HttpStatus,
 } from '@nestjs/common';
 import { Response } from 'express';
 

src/modules/inputs/inputs.controller.ts

@@ -7,6 +7,7 @@ import {
   Param,
   Patch,
   Post,
+  UseFilters,
 } from '@nestjs/common';
 import { InputsService } from './inputs.service';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
@@ -26,8 +27,11 @@ import {
 import { UpdateInputRequest } from './dtos/old_interfaces';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
+import ErrorBuilder from 'src/utils/ErrorBuilder';
+import ErrorCodes from 'src/utils/errorCodes';
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 
-@ApiTags('inputs')
+@ApiTags('Inputs')
 @Controller('inputs')
 @AuthenticateCondition((req, user) => {
   switch (req.method) {
@@ -38,7 +42,9 @@ import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entitie
         user.permissions.includes(
           PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE.seqid,
         ) ||
-        user.permissions.includes(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid)
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid,
+        )
       );
 
     default:
@@ -49,10 +55,13 @@ import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entitie
         user.permissions.includes(
           PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE.seqid,
         ) ||
-        user.permissions.includes(PERMISSIONS_GROUPS.PIPELINE.permissions.GET.seqid)
+        user.permissions.includes(
+          PERMISSIONS_GROUPS.PIPELINE.permissions.GET.seqid,
+        )
       );
   }
 })
+@UseFilters(new GrpcToHttpExceptionFilter())
 export class InputsController {
   logger: DadosferaLogger;
   constructor(
@@ -90,6 +99,7 @@ export class InputsController {
   @Post('/test-connection/get-columns')
   @ApiOkResponse({ type: TestConnectionGetColumnsRes })
   async getColumns(@Body() data: TestConnectionGetColumnsReq) {
+    throw new ErrorBuilder(ErrorCodes.NOT_IMPLEMENTED);
     this.logger.info(
       `/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`,
       {

src/modules/inputs/inputs.service.ts

@@ -72,17 +72,6 @@ export class InputsService {
       objectCamelToSnake(createInputResponse);
       return createInputResponse;
     },
-    findOne: async (data: IIdRequest) => {
-      this.logger.info('InputClientService - FindOne');
-      const findOneInputResponse = await lastValueFrom(
-        this.inputReadService.InputFindOne(data),
-      ).catch((e) => {
-        this.logger.error(e.details);
-        throw new HttpException(e.details, HttpStatus.INTERNAL_SERVER_ERROR);
-      });
-      objectCamelToSnake(findOneInputResponse);
-      return findOneInputResponse;
-    },
     update: async (updateInputDTO: UpdateInputRequest) => {
       this.logger.info('InputClientService - Update');
       const updateInputResponse = await lastValueFrom(
@@ -91,14 +80,6 @@ export class InputsService {
 
       return updateInputResponse;
     },
-    remove: async (idRequest: IIdRequest) => {
-      this.logger.info('InputClientService - Remove');
-      const removeInputResponse = await lastValueFrom(
-        this.inputWriteService.InputRemove(idRequest),
-      );
-
-      return removeInputResponse;
-    },
     testConnection: async (data: TestConnectionRequest) => {
       this.logger.info('InputClientService - TestConnection');
       const testConnectionResponse = await lastValueFrom(
@@ -239,17 +220,13 @@ export class InputsService {
   }
 
   async findOne(idRequest: IIdRequest) {
-    try {
-      const findOneInputResponse: any = await this.OLD_inputClient.findOne(
-        idRequest,
-      );
-      findOneInputResponse.input = this.adjustInputPayload(
-        findOneInputResponse.input,
-      );
-      return findOneInputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
+    const findOneInputResponse: any = await lastValueFrom(
+      this.inputReadService.InputFindOne(idRequest),
+    );
+    findOneInputResponse.input = this.adjustInputPayload(
+      findOneInputResponse.input,
+    );
+    return findOneInputResponse;
   }
 
   async update(id: string, data, info: Info) {
@@ -271,13 +248,7 @@ export class InputsService {
   }
 
   async remove(idRequest: IIdRequest) {
-    try {
-      const removeInputResponse = await this.OLD_inputClient.remove(idRequest);
-
-      return removeInputResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
+    return lastValueFrom(this.inputWriteService.InputRemove(idRequest));
   }
 
   @Timeout(60000 * 10) // Timeout set for 10 minutes

src/modules/pipelinesV2/pipelines.controller.ts

@@ -9,8 +9,15 @@ import {
   Put,
   Headers,
   Query,
+  UseFilters,
+  HttpCode,
+  HttpStatus,
 } from '@nestjs/common';
-import { ApiCreatedResponse, ApiTags } from '@nestjs/swagger';
+import {
+  ApiCreatedResponse,
+  ApiNoContentResponse,
+  ApiTags,
+} from '@nestjs/swagger';
 import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { PipelinesService } from './pipelines.service';
@@ -21,7 +28,10 @@ import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
 
 import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
 import { ICreatePipelineV2Req, IPipelineV2 } from './interfaces';
-@ApiTags('Pipelines')
+import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
+
+@UseFilters(new GrpcToHttpExceptionFilter())
+@ApiTags('PipelinesV2')
 @Controller('pipelinesV2')
 @AuthenticateCondition((req, user) => {
   let action;
@@ -143,13 +153,16 @@ export class PipelinesController {
     const result = await this.pipelinesClientService
       .findOne({ id }, metadata)
       .then((res) => {
+        //{pipeline:{tables: {tables: [], input_id: ''}}}
+        let tables = JSON.parse(res.pipeline.config.tables);
+        if (tables?.tables) tables = tables?.tables;
         Object.assign(res.pipeline, {
           transformations: res.pipeline.transformations
             ? JSON.parse(res.pipeline.transformations)
             : [],
           config: {
             cron: res.pipeline.config.cron,
-            tables: JSON.parse(res.pipeline.config.tables),
+            tables,
           },
           properties: res.pipeline.properties
             ? JSON.parse(res.pipeline.properties)
@@ -180,17 +193,17 @@ export class PipelinesController {
   }
 
   @Delete(':id')
-  async delete(@Body() data, @Param() params, @User() user: RequestUser) {
+  @ApiNoContentResponse()
+  @HttpCode(HttpStatus.NO_CONTENT)
+  async delete(@Param() params, @User() user: RequestUser) {
+    this.logger.info('PipelinesController - delete', { user });
     const { id } = params;
-    this.logger.info('PipelinesController - findOne', { user });
-
-    const metadata = PackTheMetadata({ ...user });
-
-    const response = await this.pipelinesClientService.remove(
-      { id, ...data },
-      metadata,
-    );
-
-    return response;
+    const metadata = PackTheMetadata({
+      customer_id: user.customer_id,
+      customer_name: user.customer_name,
+      user_id: user.user_id,
+    });
+    await this.pipelinesClientService.remove({ id, metadata, user });
+    this.logger.info('PipelinesController - delete: OK');
   }
 }

src/modules/pipelinesV2/pipelines.module.ts

@@ -9,6 +9,8 @@ import { PipelinesClientConfiguration } from './pipelines-client';
 
 import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipelines.module';
 import { ConnectorModule } from '../connector/connector.module';
+import { InputsModule } from '../inputs/inputs.module';
+import { TransformationsModule } from '../transformations/transformations.module';
 
 const client = new PipelinesClientConfiguration();
 
@@ -17,6 +19,8 @@ const client = new PipelinesClientConfiguration();
     ClientsModule.register([client.providerOptions]),
     OldPipelineModule,
     ConnectorModule,
+    InputsModule,
+    TransformationsModule,
   ],
   controllers: [PipelinesController],
   providers: [PipelinesService, DadosferaLogger],

src/modules/pipelinesV2/pipelines.service.ts

@@ -1,6 +1,5 @@
 import {
   BadRequestException,
-  HttpException,
   Inject,
   InternalServerErrorException,
   OnModuleInit,
@@ -20,6 +19,9 @@ import { ICreatePipelineV2Req } from './interfaces';
 import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
 import { Metadata } from '@grpc/grpc-js';
 import { ConnectorClientService } from '../connector/client.service';
+import { InputsService } from '../inputs/inputs.service';
+import { RequestUser } from 'src/authentication/user.decorator';
+import { TransformationsService } from '../transformations/transformations.service';
 
 export class PipelinesService implements OnModuleInit {
   logger: DadosferaLogger;
@@ -31,6 +33,8 @@ export class PipelinesService implements OnModuleInit {
     @Inject(PipelinesClientConfiguration.name)
     private readonly grpcClient: ClientGrpc,
     private readonly connectorService: ConnectorClientService,
+    private readonly inputsService: InputsService,
+    private readonly transformationsService: TransformationsService,
   ) {
     this.logger = dadosferaLogger.logger;
   }
@@ -129,10 +133,7 @@ export class PipelinesService implements OnModuleInit {
 
     const findOnePipelineResponse = await lastValueFrom(
       this.pipelineReadService.PipelineV2FindOne(data, metadata),
-    ).catch((err) => {
-      this.logger.error(err.message);
-      throw new Error(err);
-    });
+    );
     this.logger.info('Done');
 
     return findOnePipelineResponse;
@@ -143,39 +144,58 @@ export class PipelinesService implements OnModuleInit {
 
     const updatePipelineResponse = await lastValueFrom(
       this.pipelineWriteService.PipelineV2Update(UpdatePipelineRequest),
-    )
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        throw new Error(err);
-      });
+    );
+    this.logger.info('Done');
 
     return updatePipelineResponse;
   }
 
-  async remove(data: Messages.PipelineV2RemoveRequest, metadata: Metadata) {
-    this.logger.info('PipelinesClientService - Remove');
+  async remove(data: { id: string; metadata: Metadata; user: RequestUser }) {
+    const { id, metadata, user } = data;
+    const info = {
+      user_id: user.user_id,
+      customer_id: user.customer_id,
+      customer: user.customer_name,
+    };
+    const { pipeline } = await lastValueFrom(
+      this.pipelineReadService.PipelineV2FindOne({ id }, metadata),
+    );
+    await lastValueFrom(
+      this.pipelineWriteService.PipelineV2Remove({ id }, metadata),
+    );
 
-    const removePipelineResponse = await lastValueFrom(
-      this.pipelineWriteService.PipelineV2Remove(data, metadata),
-    )
-      .then((res) => {
-        this.logger.info('Done');
-        return res;
-      })
-      .catch((err) => {
-        this.logger.error(err.message);
-        console.log(err);
-        throw new HttpException(
-          err.details,
-          err.details === 'Erro ao pausar pipeline na plataforma.' ? 409 : 500,
+    //{pipeline:{tables: {tables: [], input_id: ''}}}
+    const input = pipeline.config.tables
+      ? JSON.parse(pipeline.config.tables)
+      : null;
+    if (input)
+      await this.inputsService
+        .remove({
+          id: input.input_id,
+          info,
+        })
+        .catch((error) =>
+          this.logger.error('Could not delete input', {
+            data: { input, error },
+          }),
         );
-      });
 
-    return removePipelineResponse;
+    const transformations: { id: string }[] = pipeline.transformations
+      ? JSON.parse(pipeline.transformations)
+      : null;
+    if (transformations && transformations.length)
+      for (const transformation of transformations) {
+        await this.transformationsService
+          .remove({
+            id: transformation.id,
+            info,
+          })
+          .catch((error) =>
+            this.logger.error('Could not delete transformation', {
+              data: { transformation, error },
+            }),
+          );
+      }
   }
 
   async findOneProperties(id: string, metadata: Metadata) {

src/utils/ErrorBuilder.ts

@@ -248,14 +248,48 @@ export function EnrichErrorCode(code: string) {
         code,
       };
 
+    case ErrorCodes.PIPELINE.NOT_FOUND:
+      return {
+        statusCode: HttpStatus.NOT_FOUND,
+        error: 'Não encontramos a pipeline solicitada',
+        message:
+          'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
+        code,
+      };
+
+    case ErrorCodes.PIPELINE.PLATFORM_NOT_FOUND:
+    case ErrorCodes.PIPELINE.PLATFORM_BAD_REQUEST:
+    case ErrorCodes.PIPELINE.PLATFORM_UNKNOWN:
+      return {
+        statusCode: HttpStatus.BAD_GATEWAY,
+        error: 'Oops... Não foi possível realizar sua ação',
+        message:
+          'Houve um erro interno ao buscar por sua pipeline. Caso o erro persista, entre em contato com o suporte',
+        code,
+      };
+    case ErrorCodes.INPUT.NOT_FOUND:
+      return {
+        statusCode: HttpStatus.NOT_FOUND,
+        error: 'Oops... O recurso que você buscou não está disponível',
+        message:
+          'Não encontramos o recurso buscado. Caso o erro persista, entre em contato com o suporte',
+        code,
+      };
+    case ErrorCodes.NOT_IMPLEMENTED:
+      return {
+        statusCode: HttpStatus.NOT_FOUND,
+        error: 'Oops... O serviço que você está tentando acessar não existe!',
+        message: 'Caso o erro persista, entre em contato com o suporte',
+        code,
+      };
     case ErrorCodes.INTERNAL:
     case ErrorCodes.UNKNOWN:
     default:
       return {
         statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
-        error: 'Desconhecido',
+        error: 'Oops... Não foi possível realizar sua ação',
         message:
-          'Erro desconhecido. Tente novamente ou entre em contato com o suporte',
+          'Ocorreu um erro desconhecido. Caso o erro persista, entre em contato com o suporte',
         code: ErrorCodes.UNKNOWN,
         details: code,
       };

src/utils/errorCodes.ts

@@ -51,17 +51,32 @@ const CONNECTION_TEST = {
   BAD_REQUEST: 'CONNECTION_TEST.BAD_REQUEST',
   UNKNOWN: 'CONNECTION_TEST.UNKNOWN',
 };
-
+const PIPELINE = {
+  NOT_FOUND: 'PIPELINE.NOT_FOUND',
+  PLATFORM_NOT_FOUND: 'PIPELINE.PLATFORM_NOT_FOUND',
+  PLATFORM_BAD_REQUEST: 'PIPELINE.PLATFORM_BAD_REQUEST',
+  PLATFORM_UNKNOWN: 'PIPELINE.PLATFORM_UNKNOWN',
+};
+const INPUT = {
+  NOT_FOUND: 'INPUT.NOT_FOUND',
+};
+const TRANSFORMATION = {
+  NOT_FOUND: 'TRANSFORMATION.NOT_FOUND',
+};
 const ErrorCodes = {
   UNKNOWN: 'UNKNOWN',
   RATE_LIMIT: 'RATE_LIMIT',
   INTERNAL: 'INTERNAL',
+  NOT_IMPLEMENTED: 'NOT_IMPLEMENTED',
   AUTH,
   ROLE,
   TERMS_OF_USE,
   USER,
   CUSTOMER,
   CONNECTION_TEST,
+  PIPELINE,
+  INPUT,
+  TRANSFORMATION,
 };
 
 export default ErrorCodes;