diff --git a/README.md b/README.md index 028ca81..4432ce9 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ # Maestro + Maestro é a API principal da Dadosfera. É responsável pela comunicação do Frontend com nossos microsserviços. ```mermaid diff --git a/docsfera.json b/docsfera.json index 0acfbbe..ef933ea 100644 --- a/docsfera.json +++ b/docsfera.json @@ -3548,6 +3548,61 @@ ] } }, + "/pipelinesV2/{pipelineId}/inputs/{id}": { + "patch": { + "operationId": "PipelinesController_updatePipelineInput", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "PipelinesV2" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/pipelinesV2/init-upload": { "post": { "operationId": "PipelinesController_initUploadFile", @@ -3960,6 +4015,1195 @@ ] } }, + "/platform/pipeline": { + "post": { + "operationId": "PlatformApiController_createPipeline", + "summary": "Create a new pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines": { + "get": { + "operationId": "PlatformApiController_getPipelines", + "summary": "List all pipelines for customer", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}": { + "get": { + "operationId": "PlatformApiController_getPipeline", + "summary": "Get pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_updatePipeline", + "summary": "Update pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "delete": { + "operationId": "PlatformApiController_deletePipeline", + "summary": "Delete pipeline by ID", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/execute": { + "post": { + "operationId": "PlatformApiController_executePipeline", + "summary": "Execute a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pause": { + "post": { + "operationId": "PlatformApiController_pausePipeline", + "summary": "Pause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/unpause": { + "post": { + "operationId": "PlatformApiController_unpausePipeline", + "summary": "Unpause a pipeline", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/memory": { + "put": { + "operationId": "PlatformApiController_updatePipelineMemory", + "summary": "Update pipeline memory configuration", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/metadata": { + "put": { + "operationId": "PlatformApiController_updatePipelineMetadata", + "summary": "Update pipeline metadata", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/metadata": { + "get": { + "operationId": "PlatformApiController_getPipelinesMetadata", + "summary": "Get all pipelines metadata", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/schemas": { + "get": { + "operationId": "PlatformApiController_getAvailableSchemas", + "summary": "Get available schemas", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipelines/catalog/tables/validate": { + "post": { + "operationId": "PlatformApiController_validateTableAndSchema", + "summary": "Validate Table and Schema", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidationTableDTO" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run": { + "get": { + "operationId": "PlatformApiController_getPipelineRuns", + "summary": "Get pipeline runs for a pipeline", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { + "get": { + "operationId": "PlatformApiController_getPipelineRun", + "summary": "Get specific pipeline run", + "parameters": [ + { + "name": "pipelineId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/pipeline/pipeline_run/{runId}/logs": { + "get": { + "operationId": "PlatformApiController_getPipelineRunLogs", + "summary": "Get pipeline run logs", + "parameters": [ + { + "name": "runId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/input": { + "put": { + "operationId": "PlatformApiController_updateJobInput", + "summary": "Update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "patch": { + "operationId": "PlatformApiController_patchJobInput", + "summary": "Partial update job input columns", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/memory": { + "put": { + "operationId": "PlatformApiController_updateJobMemory", + "summary": "Update job memory configuration", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/{jobId}/reset-state": { + "post": { + "operationId": "PlatformApiController_resetJobState", + "summary": "Reset job state", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}": { + "get": { + "operationId": "PlatformApiController_getJdbcJob", + "summary": "Get JDBC job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateJdbcSyncMode", + "summary": "Update JDBC job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/jdbc/configs/allowed_datatypes": { + "get": { + "operationId": "PlatformApiController_getJdbcAllowedDatatypes", + "summary": "Get allowed datatypes for JDBC", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}": { + "get": { + "operationId": "PlatformApiController_getSingerJob", + "summary": "Get Singer job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/singer/{jobId}/sync-mode": { + "post": { + "operationId": "PlatformApiController_updateSingerSyncMode", + "summary": "Update Singer job sync mode", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/jobs/s3/{jobId}": { + "get": { + "operationId": "PlatformApiController_getS3Job", + "summary": "Get S3 job details", + "parameters": [ + { + "name": "jobId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/platform/health": { + "get": { + "operationId": "PlatformApiController_healthCheck", + "summary": "Platform API health check", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Platform API" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/mfa": { + "post": { + "operationId": "CustomersController_enableMfaEnforce", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnforceMfa" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/links": { + "get": { + "operationId": "CustomersController_getCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinksResponse" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_setCustomerLinks", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CustomerLinkRequest" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/token": { + "get": { + "operationId": "CustomersController_getCustomerToken", + "parameters": [ + { + "name": "exp", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/monitoring-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMonitoringDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/logs-dashboard": { + "get": { + "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/access-dashboard": { + "get": { + "operationId": "CustomersController_getAccessDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/{id}/organization-info": { + "get": { + "operationId": "CustomersController_getOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_updateOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information updated" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog": { "get": { "operationId": "CatalogController_searchCatalog", @@ -5371,301 +6615,6 @@ ] } }, - "/customers/{id}/mfa": { - "post": { - "operationId": "CustomersController_enableMfaEnforce", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/EnforceMfa" - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/{id}/links": { - "get": { - "operationId": "CustomersController_getCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinksResponse" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "put": { - "operationId": "CustomersController_setCustomerLinks", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CustomerLinkRequest" - } - } - } - }, - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/token": { - "get": { - "operationId": "CustomersController_getCustomerToken", - "parameters": [ - { - "name": "exp", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "text/plain": { - "schema": { - "type": "string" - } - } - } - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/monitoring-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMonitoringDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/logs-dashboard": { - "get": { - "operationId": "CustomersController_getCustomerMixPanelLogsDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/access-dashboard": { - "get": { - "operationId": "CustomersController_getAccessDashboard", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/customers/{id}/organization-info": { - "get": { - "operationId": "CustomersController_getOrganizationInfo", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Organization information" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - }, - "put": { - "operationId": "CustomersController_updateOrganizationInfo", - "parameters": [ - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "Organization information updated" - } - }, - "tags": [ - "Customers" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, "/catalog/data-asset/share/{id}": { "get": { "operationId": "ShareController_getShareDataAsset", @@ -6937,930 +7886,6 @@ } } }, - "/platform/pipeline": { - "post": { - "operationId": "PlatformApiController_createPipeline", - "summary": "Create a new pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines": { - "get": { - "operationId": "PlatformApiController_getPipelines", - "summary": "List all pipelines for customer", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}": { - "get": { - "operationId": "PlatformApiController_getPipeline", - "summary": "Get pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_updatePipeline", - "summary": "Update pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "delete": { - "operationId": "PlatformApiController_deletePipeline", - "summary": "Delete pipeline by ID", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/execute": { - "post": { - "operationId": "PlatformApiController_executePipeline", - "summary": "Execute a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pause": { - "post": { - "operationId": "PlatformApiController_pausePipeline", - "summary": "Pause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/unpause": { - "post": { - "operationId": "PlatformApiController_unpausePipeline", - "summary": "Unpause a pipeline", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/memory": { - "put": { - "operationId": "PlatformApiController_updatePipelineMemory", - "summary": "Update pipeline memory configuration", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/metadata": { - "put": { - "operationId": "PlatformApiController_updatePipelineMetadata", - "summary": "Update pipeline metadata", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/metadata": { - "get": { - "operationId": "PlatformApiController_getPipelinesMetadata", - "summary": "Get all pipelines metadata", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables": { - "get": { - "operationId": "PlatformApiController_getAvailableTables", - "summary": "Get all tables available", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/schemas": { - "get": { - "operationId": "PlatformApiController_getAvailableSchemas", - "summary": "Get all schemas available", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipelines/catalog/tables/validate": { - "post": { - "operationId": "PlatformApiController_validateTableAndSchema", - "summary": "Validate tables and schemas", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run": { - "get": { - "operationId": "PlatformApiController_getPipelineRuns", - "summary": "Get pipeline runs for a pipeline", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/{pipelineId}/pipeline_run/{runId}": { - "get": { - "operationId": "PlatformApiController_getPipelineRun", - "summary": "Get specific pipeline run", - "parameters": [ - { - "name": "pipelineId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/pipeline/pipeline_run/{runId}/logs": { - "get": { - "operationId": "PlatformApiController_getPipelineRunLogs", - "summary": "Get pipeline run logs", - "parameters": [ - { - "name": "runId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/input": { - "put": { - "operationId": "PlatformApiController_updateJobInput", - "summary": "Update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "patch": { - "operationId": "PlatformApiController_patchJobInput", - "summary": "Partial update job input columns", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/memory": { - "put": { - "operationId": "PlatformApiController_updateJobMemory", - "summary": "Update job memory configuration", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/{jobId}/reset-state": { - "post": { - "operationId": "PlatformApiController_resetJobState", - "summary": "Reset job state", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}": { - "get": { - "operationId": "PlatformApiController_getJdbcJob", - "summary": "Get JDBC job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateJdbcSyncMode", - "summary": "Update JDBC job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/jdbc/configs/allowed_datatypes": { - "get": { - "operationId": "PlatformApiController_getJdbcAllowedDatatypes", - "summary": "Get allowed datatypes for JDBC", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}": { - "get": { - "operationId": "PlatformApiController_getSingerJob", - "summary": "Get Singer job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/singer/{jobId}/sync-mode": { - "post": { - "operationId": "PlatformApiController_updateSingerSyncMode", - "summary": "Update Singer job sync mode", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/jobs/s3/{jobId}": { - "get": { - "operationId": "PlatformApiController_getS3Job", - "summary": "Get S3 job details", - "parameters": [ - { - "name": "jobId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/platform/health": { - "get": { - "operationId": "PlatformApiController_healthCheck", - "summary": "Platform API health check", - "parameters": [], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Platform API" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/storage-explorer/tables/validate-name": { "post": { "operationId": "StorageExplorerController_validateTableName", @@ -10133,6 +10158,15 @@ "$ref": "#/components/schemas/Column" } }, + "identifier_columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "destination": { + "type": "object" + }, "type": { "type": "string" }, @@ -10149,6 +10183,12 @@ "required": [ "name", "columns", +<<<<<<< feat/sidebar-menu +======= + "references", + "identifier_columns", + "destination", +>>>>>>> beta "type" ] }, @@ -10428,6 +10468,81 @@ "engine" ] }, + "ValidationTableDTO": { + "type": "object", + "properties": { + "tables": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "tables" + ] + }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, + "CustomerLink": { + "type": "object", + "properties": { + "href": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "iconSrc": { + "type": "string" + } + }, + "required": [ + "href", + "name", + "description" + ] + }, + "CustomerLinksResponse": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLink" + } + } + }, + "required": [ + "links" + ] + }, + "CustomerLinkRequest": { + "type": "object", + "properties": { + "links": { + "type": "array", + "items": { + "$ref": "#/components/schemas/CustomerLink" + } + } + }, + "required": [ + "links" + ] + }, "IRole": { "type": "object", "properties": { @@ -11017,6 +11132,7 @@ "updated_by" ] }, +<<<<<<< feat/sidebar-menu "EnforceMfa": { "type": "object", "properties": { @@ -11151,6 +11267,8 @@ "links" ] }, +======= +>>>>>>> beta "CreateShareMetadataDto": { "type": "object", "properties": { diff --git a/package-lock.json b/package-lock.json index 615ef97..526a96d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,11 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", +<<<<<<< feat/sidebar-menu "@dadosfera/protospack-v2": "^3.38.0-beta.29", +======= + "@dadosfera/protospack-v2": "3.38.0-beta.28", +>>>>>>> beta "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1748,16 @@ } }, "node_modules/@dadosfera/protospack-v2": { +<<<<<<< feat/sidebar-menu "version": "3.38.0-beta.29", "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.29.tgz", "integrity": "sha512-1TUHKoSi2h/Pt+YEno9KOj8Zv2uAjT4qPS3xK26Tk2Wxg/OTlbxevPc+iwy8EI6YsT24WsbZfkb5JfKRlvKFHA==", +======= + "version": "3.38.0-beta.28", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.28.tgz", + "integrity": "sha512-w3Au0qschqZJ6OSHVDKaR2KeeCF2ncuJ4k7iZQceOLo1zxUU3TpDyLYyp/rp1DoSG763uIgqm3kJUXv7RlmfNw==", + "license": "ISC", +>>>>>>> beta "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 9a13d42..5b727d6 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,11 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", +<<<<<<< feat/sidebar-menu "@dadosfera/protospack-v2": "^3.38.0-beta.29", +======= + "@dadosfera/protospack-v2": "3.38.0-beta.28", +>>>>>>> beta "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/main.ts b/src/main.ts index 1085fb3..85e460a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -111,3 +111,4 @@ function configureSwagger(app: INestApplication) { ); } bootstrap(); + diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 385d60c..f54c21c 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -478,6 +478,7 @@ export class AuthController { @Get('me') async getMe(@Req() req: Request, @Res() res: Response) { this.logger.info('GET /auth/me ') + this.logger.info(JSON.stringify(req.headers)); // Check for API key header first const apiKey = req.get('X-Api-key'); diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index 89dcf84..ebb888c 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -734,6 +734,64 @@ class CatalogService implements OnModuleInit { } } + async renameTableOnNimbus( + nimbusUrl: string, + nimbusId: number, + changes: { table_name?: string; table_schema?: string; display_name?: string }, + ): Promise { + const endpoint = `${nimbusUrl}/api/catalog/table-metadata/${nimbusId}`; + this.logger.info(`Renaming table-metadata ${nimbusId} on Nimbus`, { endpoint, changes }); + await axios.patch(endpoint, changes); + } + + async renameColumnMetadataOnNimbus( + nimbusUrl: string, + databaseName: string, + oldTableName: string, + oldTableSchema: string, + newTableName: string, + newTableSchema: string, + ): Promise { + const listEndpoint = `${nimbusUrl}/api/catalog/column-metadata/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`; + this.logger.info(`Fetching column-metadata records to rename`, { listEndpoint }); + const { data: columns } = await axios.get(listEndpoint); + + const filtered = Array.isArray(columns) ? columns : []; + + for (const column of filtered) { + const patchEndpoint = `${nimbusUrl}/api/catalog/column-metadata/${column.id}`; + await axios.patch(patchEndpoint, { + table_name: newTableName, + table_schema: newTableSchema, + }); + } + this.logger.info(`Renamed ${filtered.length} column-metadata records on Nimbus`); + } + + async renameDataPreviewOnNimbus( + nimbusUrl: string, + databaseName: string, + oldTableName: string, + oldTableSchema: string, + newTableName: string, + newTableSchema: string, + ): Promise { + const listEndpoint = `${nimbusUrl}/api/catalog/data-preview/?database_name=${encodeURIComponent(databaseName)}&table_name=${encodeURIComponent(oldTableName)}&table_schema=${encodeURIComponent(oldTableSchema)}`; + this.logger.info(`Fetching data-preview records to rename`, { listEndpoint }); + const { data: previews } = await axios.get(listEndpoint); + + const filtered = Array.isArray(previews) ? previews : []; + + for (const preview of filtered) { + const patchEndpoint = `${nimbusUrl}/api/catalog/data-preview/${preview.id}`; + await axios.patch(patchEndpoint, { + table_name: newTableName, + table_schema: newTableSchema, + }); + } + this.logger.info(`Renamed ${filtered.length} data-preview records on Nimbus`); + } + async catalogDatasetItem(table_metadata_id: number, metadata: Metadata) { const customer_name_raw = metadata.get('customer_name'); diff --git a/src/modules/inputs/dtos/input.model.ts b/src/modules/inputs/dtos/input.model.ts index ef558be..504e39a 100644 --- a/src/modules/inputs/dtos/input.model.ts +++ b/src/modules/inputs/dtos/input.model.ts @@ -14,6 +14,13 @@ export class TableColumns { @ApiPropertyOptional({ type: [Column] }) references: Column[]; @ApiProperty() + identifier_columns: string[]; + @ApiProperty() + destination: Record<'raw' | 'qualify', { + table_name: string; + table_schema: string; + }> | null; + @ApiProperty() type: string; @ApiPropertyOptional({ type: [String] }) identifier_columns?: string[]; diff --git a/src/modules/inputs/inputs.service.ts b/src/modules/inputs/inputs.service.ts index d258215..cb9d25d 100644 --- a/src/modules/inputs/inputs.service.ts +++ b/src/modules/inputs/inputs.service.ts @@ -205,7 +205,7 @@ export class InputsService { } async update(id: string, data, info: Info) { - this.validateCron({ ...data, info }); + // this.validateCron({ ...data, info }); try { const updateInputResponse: any = await this.OLD_inputClient.update({ id, diff --git a/src/modules/pipelinesV2/interfaces.ts b/src/modules/pipelinesV2/interfaces.ts index db9a201..252b94c 100644 --- a/src/modules/pipelinesV2/interfaces.ts +++ b/src/modules/pipelinesV2/interfaces.ts @@ -1,6 +1,15 @@ import { ApiProperty, ApiPropertyOptional, OmitType } from '@nestjs/swagger'; import { Info } from '@dadosfera/protospack/dist/lib/interfaces'; +export class PipelineInputsDTO { + @ApiProperty() + tables: Array<{ + name: string, + type: string, + + }> +} + export class IPipelineV2 { @ApiProperty() id: string; @@ -123,3 +132,30 @@ export class PipelineFindAllReq { @ApiPropertyOptional() type?: string | undefined; } + +export interface UpdateTableDTO { + name: string; + type: string; + columns: string[]; + destinations: { + raw: { + table_schema: string; + table_name: string; + }; + qualify: { + table_schema: string; + table_name: string; + }; + }; + identifier_columns: string[]; + reference_column: { + name: string; + type: string; + }; + memory: number; +} + +export interface UpdatePlatformInputRequest { + cron: string; + tables: Array; +} diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 4f5ff1d..11e7b10 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -43,11 +43,15 @@ import { IPipelineV2, IInitUploadCSVFile, PipelineFindAllReq, + UpdatePlatformInputRequest, } from './interfaces'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; +import { TableColumns } from '../inputs/dtos/input.model'; +import { UpdateInputRequest } from '../inputs/dtos/old_interfaces'; +import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; @ApiTags('PipelinesV2') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -223,6 +227,7 @@ export class PipelinesController { .then((res) => { //{pipeline:{tables: {tables: [], input_id: ''}}} let tables = JSON.parse(res.pipeline.config.tables); + const input_id = tables?.input_id; if (tables?.tables) tables = tables.tables; Object.assign(res.pipeline, { transformations: res.pipeline.transformations @@ -231,6 +236,7 @@ export class PipelinesController { config: { cron: res.pipeline.config.cron, tables, + input_id }, properties: res.pipeline.properties ? JSON.parse(res.pipeline.properties) @@ -277,6 +283,45 @@ export class PipelinesController { return response; } + @Patch('/:pipelineId/inputs/:id') + @RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + async updatePipelineInput( + @Language() language: LanguageEnum, + @Body() pipelineInputDTO: UpdatePlatformInputRequest, + @Param('id') inputId: string, + @Param('pipelineId') pipelineId: string, + @User() user: RequestUser, + ) { + this.logger.info('PipelinesController - update', { user }); + + + const { customer_id, customer_name, user_id, username } = user; + const info: Info = { + user_id: user.user_id, + customer: user.customer_name, + customer_id: user.customer_id, + }; + const metadata = PackTheMetadata({ + customer_id, + customer_name, + user_id, + username, + language, + }); + + const response = await this.pipelinesClientService.updatePipelineInput( + pipelineId, + inputId, + pipelineInputDTO, + info, + user, + metadata, + ); + + this.logger.info('PipelinesController - update: OK', { user }); + return response; + } + @ApiInternalOnlyEndpoint() @Put('/:id') @ApiOperation({ diff --git a/src/modules/pipelinesV2/pipelines.module.ts b/src/modules/pipelinesV2/pipelines.module.ts index 7bc7cc2..9550aeb 100644 --- a/src/modules/pipelinesV2/pipelines.module.ts +++ b/src/modules/pipelinesV2/pipelines.module.ts @@ -11,6 +11,7 @@ import { PipelinesModule as OldPipelineModule } from 'src/modules/pipelines/pipe import { ConnectorModule } from '../connector/connector.module'; import { InputsModule } from '../inputs/inputs.module'; import { TransformationsModule } from '../transformations/transformations.module'; +import { PlatformApiModule } from '../platform-api/platform-api.module'; const client = new PipelinesClientConfiguration(); @@ -21,6 +22,7 @@ const client = new PipelinesClientConfiguration(); ConnectorModule, InputsModule, TransformationsModule, + PlatformApiModule ], controllers: [PipelinesController], providers: [PipelinesService, DadosferaLogger], diff --git a/src/modules/pipelinesV2/pipelines.service.ts b/src/modules/pipelinesV2/pipelines.service.ts index f10b8ee..e23ce23 100644 --- a/src/modules/pipelinesV2/pipelines.service.ts +++ b/src/modules/pipelinesV2/pipelines.service.ts @@ -1,3 +1,4 @@ +/* eslint-disable no-async-promise-executor */ import { BadRequestException, HttpException, @@ -16,7 +17,7 @@ import { lastValueFrom } from 'rxjs'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PipelinesClientConfiguration } from './pipelines-client'; -import { ICreatePipelineV2Req } from './interfaces'; +import { ICreatePipelineV2Req, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces'; import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages'; import { Metadata } from '@grpc/grpc-js'; import { ConnectorClientService } from '../connector/client.service'; @@ -26,6 +27,8 @@ import { TransformationsService } from '../transformations/transformations.servi import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath'; import ErrorCodes from 'src/utils/errorCodes'; import ErrorBuilder from 'src/utils/ErrorBuilder'; +import { PlatformApiService } from '../platform-api/platform-api.service'; +import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities'; export class PipelinesService implements OnModuleInit { logger: DadosferaLogger; @@ -39,6 +42,7 @@ export class PipelinesService implements OnModuleInit { private readonly connectorService: ConnectorClientService, private readonly inputsService: InputsService, private readonly transformationsService: TransformationsService, + private readonly platformAPI: PlatformApiService ) { this.logger = dadosferaLogger.logger; } @@ -138,6 +142,7 @@ export class PipelinesService implements OnModuleInit { const findOnePipelineResponse = await lastValueFrom( this.pipelineReadService.PipelineV2FindOne(data, metadata), ); + console.log('pipeline find one response', findOnePipelineResponse); this.logger.info('Done'); return findOnePipelineResponse; @@ -339,4 +344,137 @@ export class PipelinesService implements OnModuleInit { return res; } + + async updatePipelineInput(pipelineId: string, inputId: string, updateInputDTO: UpdatePlatformInputRequest, info: Info, user: RequestUser, metadata: Metadata) { + this.logger.info('InputClientService - Update'); + + this.logger.info('Update Dynamo Reference'); + const pipelineIdFormat = pipelineId.split('-').join('_'); + const updateInputResponse = await this.inputsService.update( + inputId, + updateInputDTO, + info + ) + + const requests = []; + + this.logger.info('Dynamo Response', updateInputResponse); + + for (const [index, table] of updateInputDTO.tables.entries()) { + const id = `${pipelineIdFormat}_${index}`; + this.logger.info('Updating input reference for table', table.name); + const body = {} + if (table.columns) { + body['column_include_list'] = table.columns; + } + + if (table.reference_column) { + body['incremental_column_name'] = table.reference_column.name; + body['incremental_column_type'] = table.reference_column.type; + } + + if (table.identifier_columns) { + body['primary_keys'] = table.identifier_columns; + } + + this.logger.info('Request body', body); + const updateCollumns = this.platformAPI.proxy( + 'PATCH', + `/jobs/${id}/input`, + user, + body + ) + requests.push(updateCollumns); + + if (table.memory) { + this.logger.info('Updating memory allocation for table', table.name); + const updateMemory = this.platformAPI.proxy( + 'PUT', + `/jobs/${id}/memory`, + user, + { + amount: table.memory + } + ) + requests.push(updateMemory); + } + + if (table.type) { + const updateSyncMode = this.updatePipelineSyncMode(table, id, user); + requests.push(updateSyncMode); + } + } + + this.logger.info('Create Platform Request for each JOB'); + + if (updateInputDTO.cron) { + const crnUpdatedRequest = new Promise(async (resolve, reject) => { + const response = await this.updatePipelineCron(updateInputDTO.cron, pipelineIdFormat, user); + + if (response.error) { + this.logger.error('Error updating pipeline cron', response.error); + return reject(new ErrorBuilder(response.error)); + } + this.logger.error('Pipeline cron updated successfully', response); + return resolve(response); + }); + requests.push(crnUpdatedRequest); + } + + this.logger.info('Executing all request for the platform api'); + + const results = await Promise.allSettled(requests); + this.logger.info('Platform api response', results); + + return updateInputResponse; + + } + + private async updatePipelineSyncMode(table: UpdateTableDTO, pipelineId: string, user: RequestUser) { + const body = { + target_load_type: table.type + } + + if (table.type === 'incremental_with_qualify') { + body['incremental_column_name'] = table.reference_column.name; + body['incremental_column_type'] = table.reference_column.type; + body['primary_keys'] = table.identifier_columns; + } + + if (table.type === 'incremental') { + body['incremental_column_name'] = table.reference_column.name; + body['incremental_column_type'] = table.reference_column.type; + } + + this.logger.info('Updating pipeline sync mode', { + pipelineId, + body + }); + + return this.platformAPI.proxy( + "POST", + `/jobs/jdbc/${pipelineId}/sync-mode`, + user, + body + ) + } + + private async updatePipelineCron(cron: string, pipelineId: string, user: RequestUser) { + try { + const response = await this.platformAPI.proxy( + 'PATCH', + `/pipeline/${pipelineId}`, + user, + { + cron + } + ); + + return response + } catch (error) { + return { + error: error.message + } + } + } } diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 978f464..044f64b 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -10,6 +10,7 @@ import { Query, Inject, BadRequestException, + HttpException, } from '@nestjs/common'; import { ApiTags, ApiOperation } from '@nestjs/swagger'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; @@ -25,6 +26,9 @@ import { ElasticsearchService } from '../../services/elasticsearch'; import { DynamoDBService, ReferenceColumn } from '../../services/dynamodb'; import { CustomersService } from '../customers/customers.service'; import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation'; +import { CatalogService } from '../catalog/catalog.service'; +import { PackTheMetadata } from '../../utils/PackTheMetadata'; +import { ValidationTableDTO } from './platform-api.dto'; type ValidateTablesDTO = { @@ -34,6 +38,11 @@ type ValidateTablesDTO = { }> } +type RenameTablesBody = { + raw?: { table_name: string; table_schema: string }; + qualify?: { table_name: string; table_schema: string }; +} + @ApiTags('Platform API') @Controller('platform') export class PlatformApiController { @@ -44,6 +53,7 @@ export class PlatformApiController { private readonly elasticsearchService: ElasticsearchService, private readonly dynamoDBService: DynamoDBService, private readonly customersService: CustomersService, + private readonly catalogService: CatalogService, @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, ) { this.logger = dadosferaLogger.logger; @@ -75,6 +85,23 @@ export class PlatformApiController { return jobId?.replace(/-/g, '_') || ''; } + private async getJobByAnyConnectorType(normalizedJobId: string, user: RequestUser): Promise { + const connectorTypes = ['jdbc', 'singer', 's3']; + for (const type of connectorTypes) { + try { + const job = await this.platformApiService.proxy( + 'GET', + `/jobs/${type}/${normalizedJobId}`, + user, + ); + return job; + } catch (error) { + // Continue to next connector type + } + } + throw new HttpException(`Job ${normalizedJobId} not found in any connector type (jdbc, singer, s3)`, 404); + } + /** * Extract the pipeline ID (base UUID) from a job ID. * Job IDs have format "uuid-suffix" where suffix is the job index (e.g., "0", "1"). @@ -728,26 +755,10 @@ export class PlatformApiController { ); } - // ==================== Catalog ROUTES ==================== - - @Get('pipelines/catalog/tables') - @ApiOperation({ summary: 'Get all tables available' }) - @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) - async getAvailableTables( - @User() user: RequestUser, - @Query() query: Record, - ) { - return this.platformApiService.proxy( - 'GET', - '/catalog/tables', - user, - undefined, - query, - ); - } + // ==================== PIPELINE VALIDATION ==================== @Get('pipelines/catalog/schemas') - @ApiOperation({ summary: 'Get all schemas available' }) + @ApiOperation({ summary: 'Get available schemas' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async getAvailableSchemas( @User() user: RequestUser, @@ -755,7 +766,7 @@ export class PlatformApiController { ) { return this.platformApiService.proxy( 'GET', - '/catalog/schemas', + `/catalog/schemas`, user, undefined, query, @@ -763,18 +774,18 @@ export class PlatformApiController { } @Post('pipelines/catalog/tables/validate') - @ApiOperation({ summary: 'Validate tables and schemas' }) - @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + @ApiOperation({ summary: 'Validate Table and Schema' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) async validateTableAndSchema( + @Body() payload: ValidationTableDTO, @User() user: RequestUser, @Query() query: Record, - @Body() validateTablesDto: ValidateTablesDTO[] ) { return this.platformApiService.proxy( 'POST', - '/catalog/tables/validate', + `/catalog/tables/validate`, user, - validateTablesDto, + payload, query, ); } @@ -968,6 +979,192 @@ export class PlatformApiController { return result; } + @Post('jobs/:jobId/rename-tables') + @ApiOperation({ summary: 'Rename job output tables and sync to catalog' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE) + async renameJobTables( + @Param('jobId') jobId: string, + @Body() body: RenameTablesBody, + @User() user: RequestUser, + ) { + const normalizedJobId = this.normalizeJobId(jobId); + + const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user); + + const result = await this.platformApiService.proxy( + 'POST', + `/jobs/${normalizedJobId}/rename-tables`, + user, + body, + ); + + try { + await this.syncTableRenameToCatalog(jobId, body, currentJob, user); + } catch (error) { + this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message }); + + const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {}); + if (reverseBody) { + try { + await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody); + this.logger.info('Snowflake rename rolled back', { jobId }); + } catch (rollbackError) { + this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message }); + } + } + + throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500); + } + + return result; + } + + private buildSnowflakeRollbackBody( + body: RenameTablesBody, + outputConfig: any, + ): RenameTablesBody | null { + const reverse: RenameTablesBody = {}; + + if (body.raw) { + const nested = outputConfig.raw; + const oldTableName = nested?.table_name || outputConfig.table_name; + const oldTableSchema = nested?.table_schema || 'PUBLIC'; + if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema }; + } + + if (body.qualify) { + const nested = outputConfig.qualify; + if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' }; + } + + return Object.keys(reverse).length > 0 ? reverse : null; + } + + /** + * Sync table rename to Elasticsearch and Nimbus. + * + * For each target (raw, qualify): + * 1. Resolve old table name from output_config + * 2. Find the ES data asset by pipeline + table + schema + * 3. Update ES, Nimbus table-metadata, column-metadata, and data-preview + * 4. If any step fails, rollback all completed steps for that target + */ + private async syncTableRenameToCatalog( + jobId: string, + body: RenameTablesBody, + currentJob: any, + user: RequestUser, + ): Promise { + const pipelineId = this.extractPipelineIdFromJobId(jobId); + const outputConfig = currentJob.output_config || {}; + const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } }); + const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`; + + const targets = this.buildRenameTargets(body, outputConfig); + + for (const { key, oldTableName, oldTableSchema, newValues } of targets) { + const rollbackSteps: Array<() => Promise> = []; + + try { + const dataAsset = await this.elasticsearchService.findDataAssetByTable( + user.customer_name, oldTableName, oldTableSchema, + ); + + if (!dataAsset) { + this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema }); + continue; + } + + const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset; + const oldValues = { table_name: oldTableName, table_schema: oldTableSchema }; + + // ES update + const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name }; + await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields); + rollbackSteps.push(() => this.elasticsearchService.updateDataAsset( + user.customer_name, esAssetId, + { name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName }, + )); + + const newTableNameUpper = newValues.table_name.toUpperCase(); + const newTableSchemaUpper = newValues.table_schema.toUpperCase(); + const oldTableNameUpper = oldTableName.toUpperCase(); + const oldTableSchemaUpper = oldTableSchema.toUpperCase(); + + // Nimbus table-metadata + if (nimbusId) { + await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper }); + rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper })); + } + + // Nimbus column-metadata + await this.catalogService.renameColumnMetadataOnNimbus( + nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper, + ); + rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus( + nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper, + )); + + // Nimbus data-preview + await this.catalogService.renameDataPreviewOnNimbus( + nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper, + ); + rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus( + nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper, + )); + + this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name }); + } catch (error) { + this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message }); + await this.executeRollback(rollbackSteps, key, jobId); + throw error; + } + } + } + + private buildRenameTargets( + body: RenameTablesBody, + outputConfig: any, + ): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> { + const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' }; + const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = []; + + for (const key of ['raw', 'qualify'] as const) { + if (!body[key]) continue; + + const nested = outputConfig[key]; + + // qualify: only sync if output_config.qualify already exists + if (key === 'qualify' && !nested?.table_name) continue; + + const oldTableName = nested?.table_name || outputConfig.table_name; + if (!oldTableName) continue; + + targets.push({ + key, + oldTableName, + oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key], + newValues: body[key], + }); + } + + return targets; + } + + private async executeRollback( + steps: Array<() => Promise>, + targetKey: string, + jobId: string, + ): Promise { + for (const rollback of steps.reverse()) { + try { + await rollback(); + } catch (error) { + this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message }); + } + } + } + @Get('jobs/jdbc/configs/allowed_datatypes') @ApiOperation({ summary: 'Get allowed datatypes for JDBC' }) @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET) diff --git a/src/modules/platform-api/platform-api.dto.ts b/src/modules/platform-api/platform-api.dto.ts new file mode 100644 index 0000000..3f6b89f --- /dev/null +++ b/src/modules/platform-api/platform-api.dto.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class ValidationTableDTO { + @ApiProperty() + tables: Array<{ + table_name: string; + table_schema: string; + }> +} \ No newline at end of file diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index c2b31e4..b78e680 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -7,9 +7,10 @@ import { PlatformApiService } from './platform-api.service'; import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; +import { CatalogModule } from '../catalog/catalog.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], diff --git a/src/modules/platform-api/platform-api.service.ts b/src/modules/platform-api/platform-api.service.ts index 97797a3..f204678 100644 --- a/src/modules/platform-api/platform-api.service.ts +++ b/src/modules/platform-api/platform-api.service.ts @@ -89,6 +89,12 @@ export class PlatformApiService { // Propagate non-2xx responses as HttpExceptions if (response.status >= 400) { + this.logger.error('Platform API upstream error', { + status: response.status, + data: response.data, + path, + method: method.toUpperCase(), + }); throw new HttpException(response.data, response.status); } diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts index e2c1857..ab2791a 100644 --- a/src/services/elasticsearch/elasticsearch.service.ts +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -358,6 +358,81 @@ export class ElasticsearchService { } } + private getDataAssetIndex(customerName: string): string { + return `${customerName}_data_assets_catalog`; + } + + async findDataAssetByTable( + customerName: string, + tableName: string, + tableSchema: string, + ): Promise<{ id: string; nimbus_id: number | null; [key: string]: any } | null> { + const index = this.getDataAssetIndex(customerName); + + this.logger.info('Elasticsearch: Searching data asset', { + index, + tableName, + tableSchema, + }); + + try { + const response = await this.client.post(`/${index}/_search`, { + query: { + bool: { + must: [ + { term: { 'table_name.keyword': tableName.toUpperCase() } }, + { term: { 'table_schema.keyword': tableSchema.toUpperCase() } }, + ], + }, + }, + size: 1, + }); + + const hits = response.data.hits?.hits || []; + if (hits.length === 0) { + this.logger.warn('Elasticsearch: Data asset not found', { tableName, tableSchema, index }); + return null; + } + + return { ...hits[0]._source, _es_id: hits[0]._id }; + } catch (error) { + this.handleError('findDataAssetByTable', error, { tableName, tableSchema, index }); + throw error; + } + } + + async updateDataAsset( + customerName: string, + assetId: string, + updates: Record, + ): Promise { + const index = this.getDataAssetIndex(customerName); + + this.logger.info('Elasticsearch: Updating data asset', { + index, + assetId, + fields: Object.keys(updates), + }); + + try { + const response = await this.client.post( + `/${index}/_update/${assetId}`, + { doc: updates }, + { params: { refresh: 'wait_for' } }, + ); + + this.logger.info('Elasticsearch: Data asset updated', { + assetId, + result: response.data.result, + }); + + return response.data; + } catch (error) { + this.handleError('updateDataAsset', error, { assetId, index }); + throw error; + } + } + private handleError( operation: string, error: any,