diff --git a/docsfera.external.json b/docsfera.external.json index d850da3..d3aa50e 100644 --- a/docsfera.external.json +++ b/docsfera.external.json @@ -534,6 +534,46 @@ ], "type": "string" } + }, + { + "name": "search", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "filters", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "size", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "page", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "type", + "required": false, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -1504,7 +1544,7 @@ } }, "info": { - "title": "Maestro - feat/docs-refactor", + "title": "Maestro - fix/upload-files", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} @@ -2173,6 +2213,9 @@ "description": { "type": "string" }, + "type": { + "type": "string" + }, "connection_id": { "type": "string" }, @@ -2213,6 +2256,7 @@ "required": [ "name", "description", + "type", "connection_id", "cron", "connector_name", @@ -2233,6 +2277,9 @@ "description": { "type": "string" }, + "type": { + "type": "string" + }, "connection_id": { "type": "string" }, @@ -2280,6 +2327,7 @@ "id", "name", "description", + "type", "connection_id", "cron", "connector_name", diff --git a/docsfera.json b/docsfera.json index dee5148..cde4eee 100644 --- a/docsfera.json +++ b/docsfera.json @@ -2707,6 +2707,46 @@ ], "type": "string" } + }, + { + "name": "search", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "filters", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "size", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "page", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "type", + "required": false, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { @@ -4676,7 +4716,7 @@ } }, "info": { - "title": "Maestro - feat/docs-refactor", + "title": "Maestro - fix/upload-files", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} @@ -6346,6 +6386,9 @@ "description": { "type": "string" }, + "type": { + "type": "string" + }, "connection_id": { "type": "string" }, @@ -6386,6 +6429,7 @@ "required": [ "name", "description", + "type", "connection_id", "cron", "connector_name", @@ -6406,6 +6450,9 @@ "description": { "type": "string" }, + "type": { + "type": "string" + }, "connection_id": { "type": "string" }, @@ -6453,6 +6500,7 @@ "id", "name", "description", + "type", "connection_id", "cron", "connector_name", @@ -6520,6 +6568,9 @@ }, "header": { "type": "boolean" + }, + "engine": { + "type": "string" } }, "required": [ @@ -6528,7 +6579,8 @@ "description", "encoding", "sep", - "header" + "header", + "engine" ] }, "IRole": { diff --git a/package-lock.json b/package-lock.json index 8cae407..664cad4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.29.0", + "@dadosfera/protospack-v2": "3.29.1", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", @@ -1727,9 +1727,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.29.0", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz", - "integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==", + "version": "3.29.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1.tgz", + "integrity": "sha512-uVswwXltdG0aX8cYVSFPRcROPWjOlW5bJGFfn5RNodKyHeBiTVDWBB3/YgocKrWV4mYBSbZ11ctBzjSAOkf/0Q==", "dependencies": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", @@ -12284,9 +12284,9 @@ } }, "@dadosfera/protospack-v2": { - "version": "3.29.0", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz", - "integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==", + "version": "3.29.1", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.1.tgz", + "integrity": "sha512-uVswwXltdG0aX8cYVSFPRcROPWjOlW5bJGFfn5RNodKyHeBiTVDWBB3/YgocKrWV4mYBSbZ11ctBzjSAOkf/0Q==", "requires": { "@grpc/grpc-js": "^1.6.7", "rxjs": "^7.5.5", diff --git a/package.json b/package.json index b829842..b2f45ee 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "dependencies": { "@aws-sdk/client-secrets-manager": "^3.112.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", - "@dadosfera/protospack-v2": "3.29.0", + "@dadosfera/protospack-v2": "3.29.1", "@grpc/grpc-js": "^1.6.7", "@grpc/proto-loader": "^0.6.13", "@nestjs/common": "^8.4.7", @@ -96,4 +96,4 @@ "tsconfig-paths": "^3.14.1", "typescript": "^4.6.3" } -} \ No newline at end of file +} diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index 9cdd9dd..37e9212 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -10,6 +10,8 @@ export class IPipelineV2 { @ApiProperty() description: string; @ApiProperty() + type: string; + @ApiProperty() connection_id: string; @ApiProperty() cron: string; @@ -104,4 +106,20 @@ export class ICreatePipelineCSVFile { @ApiProperty() header: boolean; + + @ApiProperty() + engine: string; +} + +export class PipelineFindAllReq { + @ApiPropertyOptional() + search?: string | undefined; + @ApiPropertyOptional() + filters?: string | undefined; + @ApiPropertyOptional() + size?: string | undefined; + @ApiPropertyOptional() + page?: string | undefined; + @ApiPropertyOptional() + type?: string | undefined; } diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index c457f33..5d5a6b7 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -40,6 +40,7 @@ import { ICreatePipelineV2Req, IPipelineV2, IInitUploadCSVFile, + PipelineFindAllReq, } from './interfaces'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { LanguageEnum } from 'src/utils/languages.enum'; @@ -115,7 +116,7 @@ export class PipelinesController { async findAll( @User() user: RequestUser, @Language() language: LanguageEnum, - @Query() data, + @Query() data: PipelineFindAllReq, ) { this.logger.info('PipelinesController - findAll', { user }); @@ -370,30 +371,37 @@ export class PipelinesController { @User() user: RequestUser, @Body() body: ICreatePipelineCSVFile, ) { - this.logger.info('/upload - Upload Connector Route'); + this.logger.info('/file - Upload Connector Route'); const metadata = PackTheMetadata({ ...user }); - const { name, sep, header, encoding, description, file_name } = body; + const { name, sep, header, encoding, description, file_name, engine } = + body; + + const file_format_params = { + ...(sep && { sep }), + ...(encoding && { encoding }), + ...(header !== undefined ? { header } : {}), + }; + const source_prefix = `${user.customer_name}/${file_name}`; - const upload_pipeline = { + const upload_pipeline: ICreatePipelineV2Req = { connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION, connector_name: 'Amazon S3', connector_plugin: 'aws_s3', connector_version: '1.0.0', - image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg', + image_url: `https://assets.dadosfera.ai/images/connectors/${engine}.svg`, name, description, transformations_ids: [], tags: [], cron: '@once', - config: { cron: '@once', tables: [] }, + type: 'upload', properties: { - engine: 'csv', + engine, source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS, source_prefix, - file_format_params: { sep, encoding, header: Boolean(header) }, - is_a_upload_csv: true, + file_format_params, }, input_id: undefined, }; diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index dcf740b..88f7a42 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -23,7 +23,7 @@ import { ConnectorClientService } from '../connector/client.service'; import { InputsService } from '../inputs/inputs.service'; import { RequestUser } from 'src/decorators/user.decorator'; import { TransformationsService } from '../transformations/transformations.service'; -import { getObjValueFromPath } from 'src/utils/ObjValueFromPath'; +import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath'; import ErrorCodes from 'src/utils/errorCodes'; import ErrorBuilder from 'src/utils/ErrorBuilder'; @@ -220,7 +220,7 @@ export class PipelinesService implements OnModuleInit { this.logger.info('PipelinesClientService - findOneProperties'); const { pipeline } = await this.findOne({ id }, metadata); - const pipelineProperties = JSON.parse(pipeline.properties); + const properties = JSON.parse(pipeline.properties); const connectorId = `${pipeline.connector_plugin}-${pipeline.connector_version}`; @@ -228,24 +228,30 @@ export class PipelinesService implements OnModuleInit { connectorId, metadata.get('language')[0].toString(), ); - const connector = JSON.parse(res.connector); - const config_controls = connector.config_controls || []; + const connector: { [key: string]: any; config_controls: any[] } = + JSON.parse(res.connector); + const config_controls = (connector.config_controls || []).filter( + (config_control) => { + const path: string = config_control.name; - for (const config_control of config_controls) { - const path: string = config_control.name; - const default_value = getObjValueFromPath({ - object: { properties: pipelineProperties }, - path, - }); - if (default_value != undefined) - config_control.default_value = default_value; - } + if (!objHasPath({ object: { properties }, path })) return false; + + const default_value = getObjValueFromPath({ + object: { properties }, + path, + }); + if (default_value != undefined) + config_control.default_value = default_value; + + return true; + }, + ); return { pipeline: { ...pipeline, config_controls, - properties: pipelineProperties, + properties: properties, }, }; } diff --git a/src/utils/ErrorBuilder.ts b/src/utils/ErrorBuilder.ts index 88570e9..8dad4b9 100644 --- a/src/utils/ErrorBuilder.ts +++ b/src/utils/ErrorBuilder.ts @@ -308,6 +308,20 @@ export function EnrichErrorCode(code: string) { 'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte', code, }; + case ErrorCodes.CATALOG.PREVIEW_TOO_BIG: + return { + statusCode: HttpStatus.INTERNAL_SERVER_ERROR, + error: 'Erro ao buscar prévia', + message: 'Ocorreu um erro devido ao tamanho da prévia do ativo.', + code, + }; + case ErrorCodes.CATALOG.METADATA_TOO_BIG: + return { + statusCode: HttpStatus.INTERNAL_SERVER_ERROR, + error: 'Erro ao buscar informações', + message: 'Ocorreu um erro devido ao tamanho das informações do ativo.', + code, + }; case ErrorCodes.INTERNAL: case ErrorCodes.UNKNOWN: default: diff --git a/src/utils/ObjValueFromPath.ts b/src/utils/ObjValueFromPath.ts index 43674bd..0bdb8e9 100644 --- a/src/utils/ObjValueFromPath.ts +++ b/src/utils/ObjValueFromPath.ts @@ -8,3 +8,18 @@ export function getObjValueFromPath(data: { object: T; path: string }): T { } return value; } + +export function objHasPath(data: { + object: T; + path: string; +}): boolean { + const { object, path } = data; + const keys = path.split('.'); + + let value = JSON.parse(JSON.stringify(object)); + for (const key of keys) { + if (!(key in value)) return false; + value = value[key]; + } + return true; +} diff --git a/src/utils/errorCodes.ts b/src/utils/errorCodes.ts index 3edb29f..91b83d8 100644 --- a/src/utils/errorCodes.ts +++ b/src/utils/errorCodes.ts @@ -70,6 +70,8 @@ const TRANSFORMATION = { }; const CATALOG = { DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND', + PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG', + METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG', }; const ErrorCodes = { UNKNOWN: 'UNKNOWN',