Merge pull request #180 from dadosfera/feature/update-pipeline-cron

feature/update-pipeline-cron

.github/workflows/deploy-manually.yml

@@ -12,6 +12,7 @@ on:
         options:
           - dev
           - stg
+          - stg2
           - prd
 
 jobs:
@@ -144,8 +145,7 @@ jobs:
           docker system df
   api_docs:
     needs: [extract_environment, semantic_release, deploy]
-    runs-on:
-      [self-hosted, "${{ needs.extract_environment.outputs.environment }}"]
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -153,17 +153,17 @@ jobs:
       - name: Extract Docs BlockId and PageId
         env:
           ENV: ${{ needs.extract_environment.outputs.environment }}
-          DEV_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
-          DEV_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
+          STG2_DOCS_BLOCK_ID: ${{ secrets.DEV_DOCS_BLOCK_ID }}
+          STG2_DOCS_PAGE_ID: ${{ secrets.DEV_DOCS_PAGE_ID }}
           STG_DOCS_BLOCK_ID: ${{ secrets.STG_DOCS_BLOCK_ID }}
           STG_DOCS_PAGE_ID: ${{ secrets.STG_DOCS_PAGE_ID }}
           PRD_DOCS_BLOCK_ID: ${{ secrets.PRD_DOCS_BLOCK_ID }}
           PRD_DOCS_PAGE_ID: ${{ secrets.PRD_DOCS_PAGE_ID }}
         shell: bash
         run: |
-          if [ $ENV == "dev" ]; then
-            echo "block_id=$DEV_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
-            echo "page_id=$DEV_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
+          if [ $ENV == "stg2" ]; then
+            echo "block_id=$STG2_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
+            echo "page_id=$STG2_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
           elif [ $ENV == "stg" ]; then
             echo "block_id=$STG_DOCS_BLOCK_ID" >> $GITHUB_OUTPUT
             echo "page_id=$STG_DOCS_PAGE_ID" >> $GITHUB_OUTPUT
@@ -175,12 +175,16 @@ jobs:
 
       - name: Generate API docs
         env:
-          ENV: ${{ needs.extract_environment.outputs.environment }}
           DOCS_URL: ${{ secrets.DOCS_URL }}
           DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
           DOCS_PAGE_ID: ${{ steps.extract_docs_info.outputs.page_id }}
           DOCS_BLOCK_ID: ${{ steps.extract_docs_info.outputs.block_id }}
+          EVENT: ${{ github.event_name }}
+          RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
         run: |
+          if [ ${EVENT} != "workflow_dispatch" ]; then
+            sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.json
+          fi
           sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.json
           sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.json
           sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.json

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.19.2",
+        "@dadosfera/protospack-v2": "3.21.0",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.19.2",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.19.2.tgz",
-      "integrity": "sha512-0+5V4zo7x36Y9xojYnktTllm41uhAttEFTT13i2sBXn4Cb6xSCX5rOyMHwRHb4TOHTpWvaO58lGjP/i0x+lpEg==",
+      "version": "3.21.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
+      "integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12330,9 +12330,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.19.2",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.19.2.tgz",
-      "integrity": "sha512-0+5V4zo7x36Y9xojYnktTllm41uhAttEFTT13i2sBXn4Cb6xSCX5rOyMHwRHb4TOHTpWvaO58lGjP/i0x+lpEg==",
+      "version": "3.21.0",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.21.0.tgz",
+      "integrity": "sha512-gHIRLSto813lagxqcphO4/xc82g/QkCZRslRhj5tJK4V69f/ScviBHB4CCRBRW5Md2Lql6zWuTV5R0uADFIiZA==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -17,7 +17,7 @@
     "start": "nest start",
     "start:dev": "nest start --watch",
     "start:debug": "nest start --debug --watch",
-    "start:prod": "node dist/main",
+    "start:prod": "npm run build && node dist/main",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
     "test": "jest",
     "test:watch": "jest --watch",
@@ -28,7 +28,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.19.2",
+    "@dadosfera/protospack-v2": "3.21.0",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",

src/app.module.ts

@@ -39,8 +39,8 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
     ConfigModule.forRoot({
       isGlobal: true,
     }),
-    ConnectorModule,
     AuthModule,
+    ConnectorModule,
     PermissionsModule,
     TermsOfUseModule,
     ConnectionModule,

src/authentication/authentication.guard.spec.ts

@@ -44,7 +44,7 @@ class NoClassAuthController {
   @Get('has-all-permissions')
   @RequireAllPermissions(
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.METABASE.permissions.OPEN,
+    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
   )
   async hasAllPermissions(@Body() body) {
     return { body };
@@ -53,7 +53,7 @@ class NoClassAuthController {
   @Get('has-some-permission')
   @RequireSomePermission(
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.METABASE.permissions.OPEN,
+    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
   )
   async hasSomePermission(@Body() body) {
     return { body };
@@ -62,7 +62,7 @@ class NoClassAuthController {
 
 @Controller('class-auth-condition')
 @AuthenticateCondition((req) => req.get('x-on-class') === 'ok')
-@RequireSomePermission(PERMISSIONS_GROUPS.METABASE.permissions.OPEN)
+@RequireSomePermission(PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE)
 class ClassAuthConditionController {
   @Get('body')
   async getBody(@Body() body) {
@@ -444,13 +444,13 @@ describe('authentication.guard', () => {
   NoClassAuthTest(tokenZ, ['zendesk']);
   ClassAuthConditionTest(tokenZ, ['zendesk']);
 
-  const tokenM = CreateToken([PERMISSIONS_GROUPS.METABASE.permissions.OPEN]);
+  const tokenM = CreateToken([PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE]);
   NoClassAuthTest(tokenM, ['metabase']);
   ClassAuthConditionTest(tokenM, ['metabase']);
 
   const tokenZM = CreateToken([
     PERMISSIONS_GROUPS.ZENDESK.permissions.OPEN,
-    PERMISSIONS_GROUPS.METABASE.permissions.OPEN,
+    PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE,
   ]);
   NoClassAuthTest(tokenZM, ['zendesk', 'metabase']);
   ClassAuthConditionTest(tokenZM, ['zendesk', 'metabase']);

src/authentication/permissions.enum.ts

@@ -297,12 +297,12 @@ export const PERMISSIONS_GROUPS = {
       },
       CREATE: {
         seqid: 19,
-        claim: 'POST /catalog',
-        usage: PermissionUsages.INTERNAL,
+        claim: 'catalog:create',
+        usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Criar atributos no catálogo',
-          'en-us': 'Create attributes',
-          'es-es': 'Crear atributos',
+          'pt-br': 'Criar Ativos',
+          'en-us': 'Create Assets',
+          'es-es': 'Crear Activos',
         },
       },
       UPDATE: {
@@ -429,14 +429,14 @@ export const PERMISSIONS_GROUPS = {
     },
   },
 
-  METABASE: {
+  ANALYZE: {
     title: {
       'pt-br': 'Analisar | Visualização',
       'en-us': 'Analyze | Visualization',
       'es-es': 'Analizar | Visualización',
     },
     permissions: {
-      OPEN: {
+      METABASE: {
         seqid: 11,
         claim: 'GET /metabase',
         usage: PermissionUsages.PUBLIC,
@@ -446,26 +446,26 @@ export const PERMISSIONS_GROUPS = {
           'es-es': 'Acceso Metabase',
         },
       },
-    },
-  },
-
-  ORCHEST: {
-    title: {
-      'pt-br': 'Analisar | Inteligência',
-      'en-us': 'Analyze | Intelligence',
-      'es-es': 'Analizar | Inteligencia',
-    },
-    permissions: {
       INTELLIGENCE: {
         seqid: 31,
         claim: 'intelligence:open',
-        usage: PermissionUsages.INTERNAL,
+        usage: PermissionUsages.PUBLIC,
         name: {
-          'pt-br': 'Acessar Orchest',
-          'en-us': 'Access Orchest',
-          'es-s': 'Acceso Orchest',
+          'pt-br': 'Acessar Módulo de Inteligência',
+          'en-us': 'Access Intelligence Module',
+          'es-s': 'Acceder Módulo de Inteligencia',
         },
       },
+    },
+  },
+
+  MODULES: {
+    title: {
+      'pt-br': 'Módulos da Dadosfera',
+      'en-us': 'Dadosfera Modules',
+      'es-es': 'Módulos de la Dadosfera',
+    },
+    permissions: {
       APP_BUILDER: {
         seqid: 32,
         claim: 'app-builder:open',
@@ -473,7 +473,7 @@ export const PERMISSIONS_GROUPS = {
         name: {
           'pt-br': 'Acessar App Builder',
           'en-us': 'Access App Builder',
-          'es-es': 'Acceso App Builder',
+          'es-es': 'Acceder App Builder',
         },
       },
       MACHINE_LEARNING: {
@@ -483,7 +483,7 @@ export const PERMISSIONS_GROUPS = {
         name: {
           'pt-br': 'Acessar Machine Learning',
           'en-us': 'Access Machine Learning',
-          'es-es': 'Acceso Machine Learning',
+          'es-es': 'Acceder Machine Learning',
         },
       },
     },
@@ -507,6 +507,25 @@ export const PERMISSIONS_GROUPS = {
       },
     },
   },
+  PROCESS: {
+    title: {
+      'pt-br': 'Processar',
+      'en-us': 'Process',
+      'es-es': 'Procesar',
+    },
+    permissions: {
+      TRANSFORMATION: {
+        seqid: 43,
+        claim: 'process:open',
+        usage: PermissionUsages.PUBLIC,
+        name: {
+          'pt-br': 'Acessar Módulo de Transformação',
+          'en-us': 'Access Transformation Module',
+          'es-es': 'Acceder Módulo de transformación',
+        },
+      },
+    },
+  },
   DADOSFERA: {
     title: {
       'pt-br': 'Usuários',

src/authentication/user.decorator.spec.ts

@@ -49,7 +49,7 @@ describe('user.decorator', () => {
   const fakeUserPayload = {
     user_id: 'd50d33c7-6c2b-463c-861f-e21667e7c125',
     username: 'super.admin',
-    permissions: [PERMISSIONS_GROUPS.METABASE.permissions.OPEN].map(
+    permissions: [PERMISSIONS_GROUPS.ANALYZE.permissions.METABASE].map(
       ({ seqid }) => seqid,
     ),
     customer_id: '9d18e8ae-24b9-41a3-9e8f-a25ce57555b11',

src/modules/auth/auth.controller.ts

@@ -10,7 +10,6 @@ import {
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import {
-  AuthSignInRequest,
   AuthRefreshAccessTokenRequest,
   AuthChangePasswordRequest,
   AuthResetPasswordRequest,
@@ -28,6 +27,7 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
 import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 import { Metadata } from '@grpc/grpc-js';
+import { AuthSignInReq, AuthSignInRes } from './dtos/login';
 
 @ApiTags('Auth')
 @UseFilters(new GrpcToHttpExceptionFilter())
@@ -43,6 +43,18 @@ export class AuthController {
     this.logger = dadosferaLogger.logger;
   }
 
+  @Post('sign-in')
+  @HttpCode(HttpStatus.OK)
+  async signIn(
+    @Body() { username, password, totp }: AuthSignInReq,
+    @Headers('Dadosfera-Lang') language: string,
+  ): Promise<AuthSignInRes> {
+    this.logger.info('/auth - SignIn');
+    const metadata = new Metadata();
+    metadata.add('language', language || 'pt-br');
+    return this.authClient.signIn({ username, password, totp }, metadata);
+  }
+
   @Post('/sso/snowflake')
   @RequireAllPermissions(PERMISSIONS_GROUPS.SNOWFLAKE.permissions.OPEN)
   @HttpCode(HttpStatus.OK)
@@ -58,18 +70,6 @@ export class AuthController {
     });
   }
 
-  @Post('sign-in')
-  @HttpCode(HttpStatus.OK)
-  async signIn(
-    @Body() { username, password, totp }: AuthSignInRequest,
-    @Headers('Dadosfera-Lang') language: string,
-  ) {
-    this.logger.info('/auth - SignIn');
-    const metadata = new Metadata();
-    metadata.add('language', language || 'pt-br');
-    return this.authClient.signIn({ username, password, totp }, metadata);
-  }
-
   @Post('refresh-access-token')
   @HttpCode(HttpStatus.OK)
   async refreshAccessToken(

src/modules/auth/dtos/login.ts

@@ -1,4 +1,8 @@
-import { ApiProperty } from '@nestjs/swagger';
+import {
+  AuthSignInRequest,
+  AuthSignInResponse,
+} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
+import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
 
 export class LoginDto {
   @ApiProperty()
@@ -10,3 +14,86 @@ export class LoginDto {
   @ApiProperty({ required: false })
   totp: string;
 }
+export class AuthTokens {
+  @ApiProperty()
+  accessToken: string;
+  @ApiProperty()
+  refreshToken: string;
+  @ApiPropertyOptional()
+  termsOfUseToken: string;
+  @ApiProperty({ deprecated: true })
+  idToken: string;
+}
+export enum MfaStatuses {
+  PENDING = 'pending',
+  NONE = 'none',
+  TOTP = 'totp',
+}
+
+export enum TouStatuses {
+  PENDING = 'pending',
+  REQUIRED = 'required',
+  OK = 'ok',
+}
+export class TermsOfUse {
+  @ApiProperty()
+  version: number;
+  @ApiProperty()
+  publicUrl: string;
+  @ApiProperty()
+  enforceDate: string;
+  @ApiProperty()
+  createdAt: string;
+}
+export class TermsOfUseStatus {
+  @ApiProperty({ enum: TouStatuses })
+  status: string;
+  @ApiPropertyOptional()
+  lastSigned?: TermsOfUse;
+  @ApiPropertyOptional()
+  next?: TermsOfUse;
+}
+export class AuthUser {
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  createdAt: string;
+}
+export class AuthCustomer {
+  @ApiProperty()
+  modules: string[];
+  @ApiProperty()
+  id: string;
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  tier: string;
+}
+
+export class AuthSignInReq implements AuthSignInRequest {
+  @ApiProperty()
+  username: string;
+  @ApiProperty()
+  password: string;
+  @ApiPropertyOptional()
+  totp: string;
+}
+
+export class AuthSignInRes implements AuthSignInResponse {
+  @ApiProperty()
+  permissions: string[];
+  @ApiProperty({ enum: MfaStatuses })
+  mfaStatus: string;
+  @ApiPropertyOptional()
+  customer?: AuthCustomer;
+  @ApiPropertyOptional()
+  user?: AuthUser;
+  @ApiPropertyOptional()
+  tokens?: AuthTokens;
+  @ApiPropertyOptional()
+  termsOfUse?: TermsOfUseStatus;
+}

src/modules/catalog/catalog.controller.ts

@@ -14,8 +14,8 @@ import {
 } from '@nestjs/common';
 import { ApiTags } from '@nestjs/swagger';
 import {
-  AuthenticateCondition,
   Authenticated,
+  RequireSomePermission,
 } from '../../authentication/authentication.decorator';
 import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
 import { CatalogService } from './catalog.service';
@@ -34,36 +34,9 @@ import {
   IUpdateDataRequest,
 } from './dtos';
 
-const catalogPermissions = PERMISSIONS_GROUPS.CATALOG.permissions;
 @ApiTags('Catalog')
 @Controller('catalog')
 @Authenticated()
-@AuthenticateCondition((req, user: RequestUser) => {
-  if (user.permissions.includes(catalogPermissions.DATA_MANAGER.seqid))
-    return true;
-
-  if (req.path.includes('/manage-permissions')) return true;
-  let action: keyof typeof catalogPermissions;
-
-  switch (req.method) {
-    case 'POST':
-    case 'PUT':
-      action = 'UPDATE';
-      break;
-
-    case 'DELETE':
-      if (req.path.includes('/comment')) action = 'UPDATE';
-      else action = req.method;
-      break;
-
-    default:
-      action = req.method as 'GET';
-  }
-
-  return user.permissions.includes(
-    PERMISSIONS_GROUPS.CATALOG.permissions[action].seqid,
-  );
-})
 export class CatalogController {
   logger: DadosferaLogger;
   constructor(
@@ -75,6 +48,10 @@ export class CatalogController {
   }
 
   @Get()
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async catalogAll(
     @User() user: RequestUser,
     @Query() query,
@@ -110,6 +87,10 @@ export class CatalogController {
   }
 
   @Get('tags')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async findAllTags(@Body() body) {
     this.logger.info(`/catalog - ON FIND ALL TAGS ROUTE`, {
       user: body.info.user_id,
@@ -129,6 +110,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAsset(
     @User() user: RequestUser,
     @Headers() headers,
@@ -183,6 +168,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/rls/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetRls(
     @User() user: RequestUser,
     @Headers() headers,
@@ -237,6 +226,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id/columns-metadata')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetColumnsMetadata(
     @User() user: RequestUser,
     @Headers('Dadosfera-Lang') language,
@@ -273,6 +266,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id/preview')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetPreview(
     @User() user: RequestUser,
     @Headers('Dadosfera-lang') language,
@@ -308,6 +305,10 @@ export class CatalogController {
   }
 
   @Get('data-asset/:id/docs')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.GET,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async getDataAssetDocs(
     @User() user: RequestUser,
     @Headers('Dadosfera-Lang') language,
@@ -343,6 +344,10 @@ export class CatalogController {
   }
 
   @Put('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async updateDataAsset(
     @User() user: RequestUser,
     @Headers('Dadosfera-Lang') language,
@@ -372,6 +377,10 @@ export class CatalogController {
   }
 
   @Post('data-asset/:id/docs')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async manageDataAssetDocs(
     @User() user: RequestUser,
     @Headers() headers,
@@ -440,6 +449,10 @@ export class CatalogController {
   }
 
   @Post()
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.CREATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async createDataAsset(
     @User() user: RequestUser,
     @Body() body: ICreateDataAsset,
@@ -460,6 +473,10 @@ export class CatalogController {
   }
 
   @Post('data-asset/:id/comment')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async commentOnDataAsset(
     @Param('id') id: string,
     @User() user: RequestUser,
@@ -482,6 +499,10 @@ export class CatalogController {
   }
 
   @Delete('data-asset/:id')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.DELETE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async deleteDataAsset(
     @Param('id') data_asset_id: string,
     @User() user: RequestUser,
@@ -504,6 +525,10 @@ export class CatalogController {
   }
 
   @Delete('data-asset/:id/comment')
+  @RequireSomePermission(
+    PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
+    PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
+  )
   async deleteComment(
     @Param('id') id: string,
     @User() user: RequestUser,

src/modules/connector/client.service.ts

@@ -84,7 +84,7 @@ export class ConnectorClientService implements OnModuleInit {
     });
     const responseParsed = JSON.parse(res.message);
     const connectorErrors = responseParsed.items
-      .filter((i) => i.update._index === 'connector' && i.update.error)
+      .filter((i) => i.update._index === 'connectors' && i.update.error)
       .map((i) => ({
         id: i.update._id,
         reason: i.update.error,
@@ -115,7 +115,8 @@ export class ConnectorClientService implements OnModuleInit {
       }));
     const connectorCreated = responseParsed.items
       .filter(
-        (i) => i.update._index === 'connector' && i.update.result === 'created',
+        (i) =>
+          i.update._index === 'connectors' && i.update.result === 'created',
       )
       .map((i) => ({
         id: i.update._id,
@@ -149,7 +150,8 @@ export class ConnectorClientService implements OnModuleInit {
       }));
     const connectorUpdated = responseParsed.items
       .filter(
-        (i) => i.update._index === 'connector' && i.update.result === 'updated',
+        (i) =>
+          i.update._index === 'connectors' && i.update.result === 'updated',
       )
       .map((i) => ({
         id: i.update._id,
@@ -183,7 +185,7 @@ export class ConnectorClientService implements OnModuleInit {
       }));
     const connectorNoop = responseParsed.items
       .filter(
-        (i) => i.update._index === 'connector' && i.update.result === 'noop',
+        (i) => i.update._index === 'connectors' && i.update.result === 'noop',
       )
       .map((i) => ({
         id: i.update._id,

src/modules/inputs/dtos/input.model.ts

@@ -1,106 +1,5 @@
 import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger';
-import { Info } from 'protospack/dist/lib/interfaces';
 
-export class GoogleAnalyticsClientSecrets {
-  @ApiProperty()
-  type: string;
-  @ApiProperty()
-  project_id: string;
-  @ApiProperty()
-  private_key_id: string;
-  @ApiProperty()
-  private_key: string;
-  @ApiProperty()
-  client_email: string;
-  @ApiProperty()
-  client_id: string;
-  @ApiProperty()
-  auth_uri: string;
-  @ApiProperty()
-  token_uri: string;
-  @ApiProperty()
-  auth_provider_x509_cert_url: string;
-  @ApiProperty()
-  client_x509_cert_url: string;
-}
-export class OauthObject {
-  @ApiPropertyOptional()
-  get_tokens_url: string;
-  @ApiPropertyOptional()
-  get_tokens_url_params: string;
-  @ApiPropertyOptional()
-  get_tokens_set_response: Record<string, any>;
-  @ApiPropertyOptional()
-  content_type: string;
-}
-export class InputOptions {
-  @ApiPropertyOptional()
-  oauth: OauthObject;
-  @ApiPropertyOptional()
-  skip_select_columns: true;
-  @ApiPropertyOptional()
-  skip_select_entities: false;
-  @ApiPropertyOptional()
-  skip_transformation: true;
-}
-export class FileFormatParams {
-  @ApiProperty()
-  sep: string;
-  @ApiProperty()
-  header: boolean;
-  @ApiProperty()
-  encoding: string;
-}
-
-export class AuthParameters {
-  @ApiProperty()
-  aws_access_key_id: string;
-  @ApiProperty()
-  aws_secret_access_key: string;
-}
-export class CredentialsJdbc {
-  @ApiPropertyOptional()
-  jdbc_user: string;
-  @ApiPropertyOptional()
-  jdbc_password: string;
-  @ApiPropertyOptional()
-  database: string;
-  @ApiPropertyOptional()
-  endpoint: string;
-  @ApiPropertyOptional()
-  port: string;
-  @ApiPropertyOptional()
-  engine: string;
-  @ApiPropertyOptional()
-  schema: string;
-}
-export class Credentials extends CredentialsJdbc {
-  @ApiProperty()
-  connection_type: string;
-
-  @ApiPropertyOptional()
-  client_aws_access_key_id: string;
-  @ApiPropertyOptional()
-  client_aws_secret_access_key: string;
-  @ApiPropertyOptional()
-  client_bucket: string;
-  @ApiPropertyOptional()
-  file_to_extract: string;
-  @ApiPropertyOptional()
-  file_format_params: FileFormatParams;
-
-  @ApiPropertyOptional()
-  view_id: string;
-  @ApiPropertyOptional()
-  client_secrets: GoogleAnalyticsClientSecrets;
-  @ApiPropertyOptional()
-  start_date: string;
-  @ApiPropertyOptional()
-  end_date: string;
-
-  @ApiPropertyOptional()
-  oauth_code: string;
-}
 export class Column {
   @ApiProperty()
   name: string;
@@ -114,6 +13,14 @@ export class TableColumns {
   columns: string[];
   @ApiProperty()
   references: Column[];
+  @ApiProperty()
+  type: string;
+}
+export class AvailableEntity {
+  @ApiProperty()
+  name: string;
+  @ApiProperty()
+  replication_methods: string[];
 }
 
 export class Input {
@@ -133,26 +40,6 @@ export class Input {
   created_at: string;
   @ApiProperty()
   updated_at: string;
-
-  //deprecated attributes (for retro compatibility)
-  @ApiPropertyOptional({ deprecated: true })
-  category: string;
-  @ApiPropertyOptional({ deprecated: true })
-  credentials: Credentials;
-  @ApiPropertyOptional({ deprecated: true })
-  options: InputOptions;
-  @ApiPropertyOptional({ deprecated: true })
-  info: Info;
-  @ApiPropertyOptional({ deprecated: true })
-  cron: string;
-  @ApiPropertyOptional({ deprecated: true })
-  source_bucket: string;
-  @ApiPropertyOptional({ deprecated: true })
-  source_prefix: string;
-  @ApiPropertyOptional({ deprecated: true })
-  auth_parameters: AuthParameters;
-  @ApiPropertyOptional({ deprecated: true })
-  file_format_params: FileFormatParams;
 }
 export class GetAvailableEntitiesReq {
   @ApiProperty()
@@ -160,43 +47,10 @@ export class GetAvailableEntitiesReq {
 }
 
 export class GetAvailableEntitiesRes {
-  @ApiProperty()
-  entities: string[];
+  @ApiProperty({ type: [AvailableEntity] })
+  entities: AvailableEntity[];
 }
 
-export class TestConnectionGetColumnsReq {
-  @ApiProperty()
-  plugin: string;
-  @ApiProperty()
-  tables: string[];
-  @ApiPropertyOptional()
-  credentials: CredentialsJdbc;
-  @ApiPropertyOptional()
-  id: string;
-  @ApiPropertyOptional()
-  info: Info;
-}
-export class TestConnectionGetColumnsRes {
-  @ApiProperty({ type: [TableColumns] })
-  tables: TableColumns[];
-}
-
-export class TestConnectionReq {
-  @ApiProperty()
-  plugin: string;
-  @ApiProperty()
-  credentials: CredentialsJdbc;
-  @ApiPropertyOptional()
-  info: Info;
-}
-export class TestConnectionRes {
-  @ApiProperty()
-  connection_state: boolean;
-  @ApiProperty()
-  total_entities: number;
-  @ApiProperty()
-  database_tables: string[];
-}
 export class CreateInputReq extends OmitType(Input, [
   'id',
   'created_at',

src/modules/inputs/inputs.controller.ts

@@ -19,16 +19,10 @@ import {
   GetAvailableEntitiesReq,
   GetAvailableEntitiesRes,
   Input,
-  TestConnectionGetColumnsReq,
-  TestConnectionGetColumnsRes,
-  TestConnectionReq,
-  TestConnectionRes,
 } from './dtos/input.model';
 import { UpdateInputRequest } from './dtos/old_interfaces';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
-import ErrorBuilder from 'src/utils/ErrorBuilder';
-import ErrorCodes from 'src/utils/errorCodes';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 
 @ApiTags('Inputs')
@@ -75,42 +69,19 @@ export class InputsController {
   @Get('available-entities/:plugin')
   @ApiOkResponse({ type: GetAvailableEntitiesRes })
   async getAvailableEntities(
-    @Body() body,
     @Param() params: GetAvailableEntitiesReq,
+    @User() user: RequestUser,
   ) {
-    const { info } = body;
     const { plugin } = params;
-    return await this.inputService.getAvailableEntities({ info, plugin });
-  }
-
-  @Post('/test-connection')
-  @ApiOkResponse({ type: TestConnectionRes })
-  async testConnection(@Body() data: TestConnectionReq) {
-    this.logger.info(`/test-connection - ON TEST CONNECTION ROUTE`, {
-      user: data.info.user_id,
-      customer: data.info.customer,
-    });
-
-    const response = await this.inputService.testConnection(data);
-
-    return response;
-  }
-
-  @Post('/test-connection/get-columns')
-  @ApiOkResponse({ type: TestConnectionGetColumnsRes })
-  async getColumns(@Body() data: TestConnectionGetColumnsReq) {
-    throw new ErrorBuilder(ErrorCodes.NOT_IMPLEMENTED);
-    this.logger.info(
-      `/test-connection/get-columns - ON TEST CONNECTION GET COLUMNS ROUTE`,
-      {
-        user: data.info.user_id,
-        customer: data.info.customer,
+    const availableEntities = await this.inputService.getAvailableEntities({
+      info: {
+        customer: user.customer_name,
+        customer_id: user.customer_id,
+        user_id: user.user_id,
       },
-    );
-
-    const response = await this.inputService.getColumns(data);
-
-    return response;
+      plugin,
+    });
+    return availableEntities;
   }
 
   @Post()

src/modules/inputs/inputs.service.ts

@@ -14,10 +14,10 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
 import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
 import { Input } from '@dadosfera/protospack-v2';
 import {
+  GetAvailableEntitiesRequest,
   InputCreateGenericRequest,
   InputCreateS3Request,
   InputNewCreateRequest,
-  TestConnectionGetColumnsRequest,
   TestConnectionRequest,
 } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
 import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
@@ -88,32 +88,6 @@ export class InputsService {
 
       return testConnectionResponse;
     },
-    getColumns: async (data: TestConnectionGetColumnsRequest) => {
-      this.logger.info('InputClientService - TestConnection/Get-Columns');
-      const getColumnsResponse = await new Promise((resolve, reject) => {
-        this.inputReadService.GetColumns(data).subscribe({
-          next(x) {
-            resolve(objectCamelToSnake(x));
-          },
-          error(err) {
-            reject(err);
-          },
-          complete() {
-            // console.log('done');
-          },
-        });
-      })
-        .then((res) => {
-          this.logger.info('Done');
-          return res;
-        })
-        .catch((err) => {
-          this.logger.error(err.message);
-          throw new Error(err);
-        });
-
-      return getColumnsResponse;
-    },
     getAvailableEntities: async (data) => {
       this.logger.info('InputClientService - GetAvailableEntities');
       data;
@@ -188,12 +162,9 @@ export class InputsService {
     // this.validateCron(data);
     const { body, info } = data;
 
-    if (body.info) delete body.info;
-
-    const inputCreateGenericRequest: any | InputCreateGenericRequest = {
+    const inputCreateGenericRequest: InputCreateGenericRequest = {
       input: {
         ...body,
-        credentials: body.credentials && JSON.stringify(body.credentials),
       },
       info,
     };
@@ -205,8 +176,8 @@ export class InputsService {
     return { input: adjustedInput };
   }
 
-  async getAvailableEntities(data): Promise<{ entities: string[] }> {
-    return await this.OLD_inputClient.getAvailableEntities(data);
+  async getAvailableEntities(data: GetAvailableEntitiesRequest) {
+    return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
   }
 
   async findAll(info: Info) {
@@ -263,17 +234,6 @@ export class InputsService {
     }
   }
 
-  async getColumns(data) {
-    try {
-      const testConnectionGetColumnsResponse =
-        await this.OLD_inputClient.getColumns(data);
-
-      return testConnectionGetColumnsResponse;
-    } catch (err) {
-      throw new HttpException(err.message, HttpStatus.NOT_FOUND);
-    }
-  }
-
   generateInputS3Payload(payload) {
     const { credentials, plugin, cron } = payload;
     if (!credentials) return payload;

src/modules/mixpanel/mixpanel.controller.ts

@@ -1,7 +1,9 @@
 import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
 import { init } from 'mixpanel';
+import { Authenticated } from 'src/authentication/authentication.decorator';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 
+@Authenticated()
 @Controller('trackEvent')
 export class MixpanelController {
   constructor(

src/modules/permissions/permissions.controller.ts

@@ -1,14 +1,8 @@
 import { Controller, Get, Headers, Inject } from '@nestjs/common';
 import { ApiHeader, ApiOkResponse, ApiTags } from '@nestjs/swagger';
 import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
-import {
-  Authenticated,
-  RequireAllPermissions,
-} from 'src/authentication/authentication.decorator';
-import {
-  PERMISSIONS_GROUPS,
-  PermissionUsages,
-} from 'src/authentication/permissions.enum';
+import { Authenticated } from 'src/authentication/authentication.decorator';
+import { PermissionUsages } from 'src/authentication/permissions.enum';
 import { RequestUser, User } from 'src/authentication/user.decorator';
 import { LanguageEnum } from 'src/utils/languages.enum';
 import { GetPublicPermissionsRes } from './dto/entities';

src/modules/pipelinesV2/pipelines.controller.ts

@@ -12,6 +12,7 @@ import {
   UseFilters,
   HttpCode,
   HttpStatus,
+  Patch,
 } from '@nestjs/common';
 import {
   ApiCreatedResponse,
@@ -45,6 +46,10 @@ import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filt
       action = 'UPDATE';
       break;
 
+    case 'PATCH':
+      action = 'UPDATE';
+      break;
+
     default:
       action = req.method;
   }
@@ -175,20 +180,49 @@ export class PipelinesController {
 
   @Put('/:id')
   async update(
+    @Headers('Dadosfera-Lang') language,
     @Body() updatePipelineDto,
     @Param('id') id,
     @User() user: RequestUser,
   ) {
-    this.logger.info('PipelinesController - findOne', { user });
+    this.logger.info('PipelinesController - update', { user });
     const { info } = updatePipelineDto;
     delete updatePipelineDto.info;
 
-    const response = await this.pipelinesClientService.update({
-      ...updatePipelineDto,
-      info,
-      id,
+    const { customer_id, customer_name, user_id, username } = user;
+    if (!language) language = 'en-us';
+
+    const metadata = PackTheMetadata({
+      customer_id,
+      customer_name,
+      user_id,
+      username,
+      language,
     });
 
+    const response = await this.pipelinesClientService.update(
+      {
+        ...updatePipelineDto,
+        info,
+        id,
+      },
+      metadata,
+    );
+
+    this.logger.info('PipelinesController - update: OK', { user });
+    return response;
+  }
+
+  @Patch('/:id')
+  async updateByPatch(
+    @Headers('Dadosfera-Lang') language,
+    @Body() updatePipelineDto,
+    @Param('id') id,
+    @User() user: RequestUser,
+  ) {
+    this.logger.info('PipelinesController - patch', { user });
+    const response = await this.update(language, updatePipelineDto, id, user);
+    this.logger.info('PipelinesController - patch: OK', { user });
     return response;
   }
 

src/modules/pipelinesV2/pipelines.service.ts

@@ -22,6 +22,7 @@ import { ConnectorClientService } from '../connector/client.service';
 import { InputsService } from '../inputs/inputs.service';
 import { RequestUser } from 'src/authentication/user.decorator';
 import { TransformationsService } from '../transformations/transformations.service';
+import { getObjValueFromPath } from 'src/utils/ObjValueFromPath';
 
 export class PipelinesService implements OnModuleInit {
   logger: DadosferaLogger;
@@ -139,11 +140,17 @@ export class PipelinesService implements OnModuleInit {
     return findOnePipelineResponse;
   }
 
-  async update(UpdatePipelineRequest: Messages.PipelineV2UpdateRequest) {
+  async update(
+    UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
+    metadata,
+  ) {
     this.logger.info('PipelinesClientService - Update');
 
     const updatePipelineResponse = await lastValueFrom(
-      this.pipelineWriteService.PipelineV2Update(UpdatePipelineRequest),
+      this.pipelineWriteService.PipelineV2Update(
+        UpdatePipelineRequest,
+        metadata,
+      ),
     );
     this.logger.info('Done');
 
@@ -213,16 +220,14 @@ export class PipelinesService implements OnModuleInit {
     const connector = JSON.parse(res.connector);
     const config_controls = connector.config_controls || [];
 
-    for (const key in pipelineProperties) {
-      const i = config_controls
-        .map((cc) => cc.name)
-        .indexOf(`properties.${key}`);
-      if (i > -1) {
-        config_controls[i] = {
-          ...config_controls[i],
-          default_value: pipelineProperties[key],
-        };
-      }
+    for (const config_control of config_controls) {
+      const path: string = config_control.name;
+      const default_value = getObjValueFromPath({
+        object: { properties: pipelineProperties },
+        path,
+      });
+      if (default_value != undefined)
+        config_control.default_value = default_value;
     }
 
     return {

src/utils/ObjValueFromPath.ts

@@ -0,0 +1,10 @@
+export function getObjValueFromPath<T>(data: { object: T; path: string }): T {
+  const { object, path } = data;
+  const keys = path.split('.');
+
+  let value = JSON.parse(JSON.stringify(object));
+  for (const key of keys) {
+    if (value != undefined) value = value[key];
+  }
+  return value;
+}

tsconfig.build.json

@@ -1,4 +1,4 @@
 {
   "extends": "./tsconfig.json",
-  "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts", "jest.config.ts"]
 }