Merge pull request #208 from dadosfera/fix/upload-files

FIX: create pipeline from any file type on upload

docsfera.external.json

@@ -534,6 +534,46 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "search",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "filters",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "size",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "page",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "type",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -1504,7 +1544,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/docs-refactor",
+      "title": "Maestro - fix/upload-files",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}
@@ -2173,6 +2213,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -2213,6 +2256,7 @@
           "required": [
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -2233,6 +2277,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -2280,6 +2327,7 @@
             "id",
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",

docsfera.json

@@ -2707,6 +2707,46 @@
                 ],
                 "type": "string"
               }
+            },
+            {
+              "name": "search",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "filters",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "size",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "page",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
+            },
+            {
+              "name": "type",
+              "required": false,
+              "in": "query",
+              "schema": {
+                "type": "string"
+              }
             }
           ],
           "responses": {
@@ -4676,7 +4716,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/docs-refactor",
+      "title": "Maestro - fix/upload-files",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}
@@ -6346,6 +6386,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -6386,6 +6429,7 @@
           "required": [
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -6406,6 +6450,9 @@
             "description": {
               "type": "string"
             },
+            "type": {
+              "type": "string"
+            },
             "connection_id": {
               "type": "string"
             },
@@ -6453,6 +6500,7 @@
             "id",
             "name",
             "description",
+            "type",
             "connection_id",
             "cron",
             "connector_name",
@@ -6520,6 +6568,9 @@
             },
             "header": {
               "type": "boolean"
+            },
+            "engine": {
+              "type": "string"
             }
           },
           "required": [
@@ -6528,7 +6579,8 @@
             "description",
             "encoding",
             "sep",
-            "header"
+            "header",
+            "engine"
           ]
         },
         "IRole": {

package-lock.json

@@ -12,7 +12,7 @@
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.112.0",
         "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-        "@dadosfera/protospack-v2": "3.29.0",
+        "@dadosfera/protospack-v2": "3.29.1",
         "@grpc/grpc-js": "^1.6.7",
         "@grpc/proto-loader": "^0.6.13",
         "@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
       }
     },
     "node_modules/@dadosfera/protospack-v2": {
-      "version": "3.29.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz",
-      "integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==",
+      "version": "3.29.1",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1.tgz",
+      "integrity": "sha512-uVswwXltdG0aX8cYVSFPRcROPWjOlW5bJGFfn5RNodKyHeBiTVDWBB3/YgocKrWV4mYBSbZ11ctBzjSAOkf/0Q==",
       "dependencies": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",
@@ -12284,9 +12284,9 @@
       }
     },
     "@dadosfera/protospack-v2": {
-      "version": "3.29.0",
-      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz",
-      "integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==",
+      "version": "3.29.1",
+      "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1.tgz",
+      "integrity": "sha512-uVswwXltdG0aX8cYVSFPRcROPWjOlW5bJGFfn5RNodKyHeBiTVDWBB3/YgocKrWV4mYBSbZ11ctBzjSAOkf/0Q==",
       "requires": {
         "@grpc/grpc-js": "^1.6.7",
         "rxjs": "^7.5.5",

package.json

@@ -29,7 +29,7 @@
   "dependencies": {
     "@aws-sdk/client-secrets-manager": "^3.112.0",
     "@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
-    "@dadosfera/protospack-v2": "3.29.0",
+    "@dadosfera/protospack-v2": "3.29.1",
     "@grpc/grpc-js": "^1.6.7",
     "@grpc/proto-loader": "^0.6.13",
     "@nestjs/common": "^8.4.7",
@@ -96,4 +96,4 @@
     "tsconfig-paths": "^3.14.1",
     "typescript": "^4.6.3"
   }
-}
+}

src/modules/pipelinesV2/interfaces.ts

@@ -10,6 +10,8 @@ export class IPipelineV2 {
   @ApiProperty()
   description: string;
   @ApiProperty()
+  type: string;
+  @ApiProperty()
   connection_id: string;
   @ApiProperty()
   cron: string;
@@ -104,4 +106,20 @@ export class ICreatePipelineCSVFile {
 
   @ApiProperty()
   header: boolean;
+
+  @ApiProperty()
+  engine: string;
+}
+
+export class PipelineFindAllReq {
+  @ApiPropertyOptional()
+  search?: string | undefined;
+  @ApiPropertyOptional()
+  filters?: string | undefined;
+  @ApiPropertyOptional()
+  size?: string | undefined;
+  @ApiPropertyOptional()
+  page?: string | undefined;
+  @ApiPropertyOptional()
+  type?: string | undefined;
 }

src/modules/pipelinesV2/pipelines.controller.ts

@@ -40,6 +40,7 @@ import {
   ICreatePipelineV2Req,
   IPipelineV2,
   IInitUploadCSVFile,
+  PipelineFindAllReq,
 } from './interfaces';
 import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
 import { LanguageEnum } from 'src/utils/languages.enum';
@@ -115,7 +116,7 @@ export class PipelinesController {
   async findAll(
     @User() user: RequestUser,
     @Language() language: LanguageEnum,
-    @Query() data,
+    @Query() data: PipelineFindAllReq,
   ) {
     this.logger.info('PipelinesController - findAll', { user });
 
@@ -370,30 +371,37 @@ export class PipelinesController {
     @User() user: RequestUser,
     @Body() body: ICreatePipelineCSVFile,
   ) {
-    this.logger.info('/upload - Upload Connector Route');
+    this.logger.info('/file - Upload Connector Route');
     const metadata = PackTheMetadata({ ...user });
 
-    const { name, sep, header, encoding, description, file_name } = body;
+    const { name, sep, header, encoding, description, file_name, engine } =
+      body;
+
+    const file_format_params = {
+      ...(sep && { sep }),
+      ...(encoding && { encoding }),
+      ...(header !== undefined ? { header } : {}),
+    };
+
     const source_prefix = `${user.customer_name}/${file_name}`;
 
-    const upload_pipeline = {
+    const upload_pipeline: ICreatePipelineV2Req = {
       connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
       connector_name: 'Amazon S3',
       connector_plugin: 'aws_s3',
       connector_version: '1.0.0',
-      image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg',
+      image_url: `https://assets.dadosfera.ai/images/connectors/${engine}.svg`,
       name,
       description,
       transformations_ids: [],
       tags: [],
       cron: '@once',
-      config: { cron: '@once', tables: [] },
+      type: 'upload',
       properties: {
-        engine: 'csv',
+        engine,
         source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
         source_prefix,
-        file_format_params: { sep, encoding, header: Boolean(header) },
-        is_a_upload_csv: true,
+        file_format_params,
       },
       input_id: undefined,
     };

src/modules/pipelinesV2/pipelines.service.ts

@@ -23,7 +23,7 @@ import { ConnectorClientService } from '../connector/client.service';
 import { InputsService } from '../inputs/inputs.service';
 import { RequestUser } from 'src/decorators/user.decorator';
 import { TransformationsService } from '../transformations/transformations.service';
-import { getObjValueFromPath } from 'src/utils/ObjValueFromPath';
+import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
 import ErrorCodes from 'src/utils/errorCodes';
 import ErrorBuilder from 'src/utils/ErrorBuilder';
 
@@ -220,7 +220,7 @@ export class PipelinesService implements OnModuleInit {
     this.logger.info('PipelinesClientService - findOneProperties');
     const { pipeline } = await this.findOne({ id }, metadata);
 
-    const pipelineProperties = JSON.parse(pipeline.properties);
+    const properties = JSON.parse(pipeline.properties);
 
     const connectorId = `${pipeline.connector_plugin}-${pipeline.connector_version}`;
 
@@ -228,24 +228,30 @@ export class PipelinesService implements OnModuleInit {
       connectorId,
       metadata.get('language')[0].toString(),
     );
-    const connector = JSON.parse(res.connector);
-    const config_controls = connector.config_controls || [];
+    const connector: { [key: string]: any; config_controls: any[] } =
+      JSON.parse(res.connector);
+    const config_controls = (connector.config_controls || []).filter(
+      (config_control) => {
+        const path: string = config_control.name;
 
-    for (const config_control of config_controls) {
-      const path: string = config_control.name;
-      const default_value = getObjValueFromPath({
-        object: { properties: pipelineProperties },
-        path,
-      });
-      if (default_value != undefined)
-        config_control.default_value = default_value;
-    }
+        if (!objHasPath({ object: { properties }, path })) return false;
+
+        const default_value = getObjValueFromPath({
+          object: { properties },
+          path,
+        });
+        if (default_value != undefined)
+          config_control.default_value = default_value;
+
+        return true;
+      },
+    );
 
     return {
       pipeline: {
         ...pipeline,
         config_controls,
-        properties: pipelineProperties,
+        properties: properties,
       },
     };
   }

src/utils/ErrorBuilder.ts

@@ -308,6 +308,20 @@ export function EnrichErrorCode(code: string) {
           'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
         code,
       };
+    case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
+      return {
+        statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
+        error: 'Erro ao buscar prévia',
+        message: 'Ocorreu um erro devido ao tamanho da prévia do ativo.',
+        code,
+      };
+    case ErrorCodes.CATALOG.METADATA_TOO_BIG:
+      return {
+        statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
+        error: 'Erro ao buscar informações',
+        message: 'Ocorreu um erro devido ao tamanho das informações do ativo.',
+        code,
+      };
     case ErrorCodes.INTERNAL:
     case ErrorCodes.UNKNOWN:
     default:

src/utils/ObjValueFromPath.ts

@@ -8,3 +8,18 @@ export function getObjValueFromPath<T>(data: { object: T; path: string }): T {
   }
   return value;
 }
+
+export function objHasPath<T extends object>(data: {
+  object: T;
+  path: string;
+}): boolean {
+  const { object, path } = data;
+  const keys = path.split('.');
+
+  let value = JSON.parse(JSON.stringify(object));
+  for (const key of keys) {
+    if (!(key in value)) return false;
+    value = value[key];
+  }
+  return true;
+}

src/utils/errorCodes.ts

@@ -70,6 +70,8 @@ const TRANSFORMATION = {
 };
 const CATALOG = {
   DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
+  PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
+  METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
 };
 const ErrorCodes = {
   UNKNOWN: 'UNKNOWN',