diff --git a/docsfera.json b/docsfera.json index a99d60e..eed1cb5 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7952,2591 +7952,6 @@ } } }, - "/storage-explorer/tables/validate-name": { - "post": { - "operationId": "StorageExplorerController_validateTableName", - "summary": "Validate table name in PostgreSQL and Snowflake", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/tables": { - "post": { - "operationId": "StorageExplorerController_createTable", - "summary": "Create a new table", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - }, - "get": { - "operationId": "StorageExplorerController_listTables", - "summary": "List all tables with pagination", - "parameters": [ - { - "name": "page", - "required": true, - "in": "query", - "schema": { - "type": "number" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/tables/{tableId}": { - "get": { - "operationId": "StorageExplorerController_getTable", - "summary": "Get table details by ID", - "parameters": [ - { - "name": "tableId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/tables/{tableId}/datasets/{datasetId}": { - "post": { - "operationId": "StorageExplorerController_linkDatasetToTable", - "summary": "Link a dataset to a table", - "parameters": [ - { - "name": "tableId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "datasetId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/tables/{tableId}/datasets": { - "get": { - "operationId": "StorageExplorerController_getTableDatasets", - "summary": "Get all datasets linked to a table", - "parameters": [ - { - "name": "tableId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/tables/{tableId}/schema": { - "get": { - "operationId": "StorageExplorerController_getTableSchema", - "summary": "Get table schema", - "parameters": [ - { - "name": "tableId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/tables/{tableId}/validate-compatibility/{datasetId}": { - "post": { - "operationId": "StorageExplorerController_validateSchemaCompatibility", - "summary": "Validate schema compatibility between table and dataset", - "parameters": [ - { - "name": "tableId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "datasetId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/datasets/{datasetId}/preview": { - "get": { - "operationId": "StorageExplorerController_getDatasetPreview", - "summary": "Get dataset preview data", - "parameters": [ - { - "name": "datasetId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "limit", - "required": true, - "in": "query", - "schema": { - "type": "number" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/datasets/{datasetId}/schema": { - "get": { - "operationId": "StorageExplorerController_getDatasetSchema", - "summary": "Get dataset schema information", - "parameters": [ - { - "name": "datasetId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - }, - { - "name": "force_refresh", - "required": true, - "in": "query", - "schema": { - "type": "boolean" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/datasets/upload/{uploadId}": { - "get": { - "operationId": "StorageExplorerController_listDatasetsByUpload", - "summary": "List all datasets for a specific upload", - "parameters": [ - { - "name": "uploadId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/datasets/{datasetId}/refresh-schema": { - "put": { - "operationId": "StorageExplorerController_refreshDatasetSchema", - "summary": "Refresh dataset schema with new parsing options (Excel)", - "parameters": [ - { - "name": "datasetId", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/storage/uploads/history": { - "get": { - "operationId": "StorageExplorerController_listFileExplorerUploads", - "summary": "List file explorer uploads with pagination", - "parameters": [ - { - "name": "page", - "required": true, - "in": "query", - "schema": { - "type": "number" - } - }, - { - "name": "limit", - "required": true, - "in": "query", - "schema": { - "type": "number" - } - }, - { - "name": "folder_path", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/storage/browse": { - "get": { - "operationId": "StorageExplorerController_browseStorage", - "summary": "Browse folders and files in storage", - "parameters": [ - { - "name": "path", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/storage/upload/batch": { - "post": { - "operationId": "StorageExplorerController_batchUpload", - "summary": "Upload multiple files to storage", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/storage/folder/create": { - "post": { - "operationId": "StorageExplorerController_createFolder", - "summary": "Create a new folder in storage", - "parameters": [], - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/storage/download": { - "get": { - "operationId": "StorageExplorerController_downloadFile", - "summary": "Download a file from storage", - "parameters": [ - { - "name": "file_path", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/storage-explorer/storage/metadata": { - "get": { - "operationId": "StorageExplorerController_getFileMetadata", - "summary": "Get detailed file metadata", - "parameters": [ - { - "name": "file_path", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } - } - }, - "tags": [ - "Storage Explorer" - ], - "security": [ - { - "access-token": [] - }, - { - "access-token": [] - } - ] - } - }, - "/health": { - "get": { - "operationId": "HealthController_check", - "parameters": [], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "Health" - ] - } - } - }, - "info": { - "title": "Maestro", - "description": "This is the Maestro API", - "version": "1.0.0", - "contact": {} - }, - "tags": [], - "servers": [], - "components": { - "securitySchemes": { - "access-token": { - "type": "apiKey", - "name": "Authorization", - "in": "header" - } - }, - "schemas": { - "AuthSignInReq": { - "type": "object", - "properties": { - "username": { - "type": "string" - }, - "password": { - "type": "string" - }, - "totp": { - "type": "string", - "description": "TOTP code used to login when Multi-Factor Authentication is enabled" - } - }, - "required": [ - "username", - "password" - ] - }, - "AuthRefreshAccessTokenReq": { - "type": "object", - "properties": { - "refreshToken": { - "type": "string" - }, - "userId": { - "type": "string" - } - }, - "required": [ - "refreshToken", - "userId" - ] - }, - "AuthRefreshAccessTokenRes": { - "type": "object", - "properties": { - "permissions": { - "type": "array", - "items": { - "type": "string" - } - }, - "accessToken": { - "type": "string" - } - }, - "required": [ - "permissions", - "accessToken" - ] - }, - "CreateApiKeyDto": { - "type": "object", - "properties": { - "permissions": { - "description": "Array of permission IDs", - "type": "array", - "items": { - "type": "number" - } - } - }, - "required": [ - "permissions" - ] - }, - "PermissionDto": { - "type": "object", - "properties": { - "id": { - "type": "number" - }, - "name": { - "type": "string" - } - }, - "required": [ - "id", - "name" - ] - }, - "CreateApiKeyResponseDto": { - "type": "object", - "properties": { - "id": { - "type": "string", - "format": "uuid" - }, - "key_mask": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PermissionDto" - } - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "created_by": { - "type": "string" - }, - "key": { - "type": "string" - } - }, - "required": [ - "id", - "key_mask", - "permissions", - "created_at", - "created_by", - "key" - ] - }, - "ApiKeyBaseResponseDto": { - "type": "object", - "properties": { - "id": { - "type": "string", - "format": "uuid" - }, - "key_mask": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PermissionDto" - } - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "created_by": { - "type": "string" - } - }, - "required": [ - "id", - "key_mask", - "permissions", - "created_at", - "created_by" - ] - }, - "CreateConnectionDto": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "type": { - "type": "string" - }, - "network_config_id": { - "type": "string" - }, - "properties": { - "type": "object" - }, - "connector_id": { - "type": "string" - }, - "connector_name": { - "type": "string" - }, - "plugin": { - "type": "string" - }, - "image_url": { - "type": "string" - } - }, - "required": [ - "name", - "description", - "type", - "properties", - "connector_id", - "connector_name", - "plugin", - "image_url" - ] - }, - "ConnectionToCatalogDto": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "type": { - "type": "string" - }, - "network_config_id": { - "type": "string" - }, - "connector_id": { - "type": "string" - }, - "connector_name": { - "type": "string" - }, - "plugin": { - "type": "string" - }, - "image_url": { - "type": "string" - } - }, - "required": [ - "name", - "description", - "type", - "connector_id", - "connector_name", - "plugin", - "image_url" - ] - }, - "ConnectionRes": { - "type": "object", - "properties": { - "connection": { - "$ref": "#/components/schemas/ConnectionToCatalogDto" - }, - "message": { - "type": "string" - } - }, - "required": [ - "connection", - "message" - ] - }, - "UpdateConnectionDto": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "network_config_id": { - "type": "string" - }, - "properties": { - "type": "object" - } - }, - "required": [ - "name", - "description", - "properties" - ] - }, - "ConnectionsRes": { - "type": "object", - "properties": { - "connections": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ConnectionToCatalogDto" - } - }, - "message": { - "type": "string" - } - }, - "required": [ - "connections", - "message" - ] - }, - "ConnectionDto": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "customer_id": { - "type": "string" - }, - "customer_name": { - "type": "string" - }, - "user_id": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "pipelines": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "type": { - "type": "string" - }, - "network_config_id": { - "type": "string" - }, - "properties": { - "type": "object" - }, - "connector_id": { - "type": "string" - }, - "connector_name": { - "type": "string" - }, - "plugin": { - "type": "string" - }, - "image_url": { - "type": "string" - } - }, - "required": [ - "id", - "customer_id", - "customer_name", - "user_id", - "updated_at", - "created_at", - "pipelines", - "name", - "description", - "type", - "properties", - "connector_id", - "connector_name", - "plugin", - "image_url" - ] - }, - "ConnectionDetailsRes": { - "type": "object", - "properties": { - "connection": { - "$ref": "#/components/schemas/ConnectionDto" - }, - "message": { - "type": "string" - } - }, - "required": [ - "connection", - "message" - ] - }, - "Customer": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "domain": { - "type": "string" - }, - "tier": { - "type": "string" - }, - "info": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - }, - "required": [ - "id", - "name", - "domain", - "tier", - "info", - "createdAt", - "updatedAt" - ] - }, - "UserRole": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PermissionDto" - } - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "isPublic": { - "type": "boolean" - }, - "customer": { - "$ref": "#/components/schemas/Customer" - }, - "usage": { - "type": "string" - }, - "canUntoggle": { - "type": "boolean", - "description": "Whether this role can be untoggled" - } - }, - "required": [ - "id", - "name", - "description", - "createdAt", - "updatedAt", - "isPublic", - "customer", - "usage" - ] - }, - "IUserByCustomer": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "roles": { - "type": "array", - "items": { - "$ref": "#/components/schemas/UserRole" - } - }, - "jobTitle": { - "type": "string" - }, - "department": { - "type": "string" - }, - "hierarchy": { - "type": "string" - }, - "bio": { - "type": "string" - }, - "companyName": { - "type": "string" - }, - "personalSite": { - "type": "string" - }, - "mfaStatus": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "status": { - "type": "string" - }, - "lastLogin": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "type": "string" - } - }, - "authProvider": { - "type": "string" - } - }, - "required": [ - "id", - "name", - "email", - "roles", - "jobTitle", - "department", - "hierarchy", - "bio", - "companyName", - "personalSite", - "mfaStatus", - "createdAt", - "updatedAt", - "status" - ] - }, - "GetAllUsersByCustomerIdRes": { - "type": "object", - "properties": { - "users": { - "type": "array", - "items": { - "$ref": "#/components/schemas/IUserByCustomer" - } - }, - "departments": { - "type": "array", - "items": { - "type": "string" - } - }, - "jobTitles": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "users", - "departments", - "jobTitles" - ] - }, - "Hierarchy": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - } - }, - "required": [ - "id", - "name" - ] - }, - "GetAllHierarchiesRes": { - "type": "object", - "properties": { - "hierarchies": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Hierarchy" - } - } - }, - "required": [ - "hierarchies" - ] - }, - "GetAllDepartmentsRes": { - "type": "object", - "properties": { - "departments": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "departments" - ] - }, - "CreateUserReq": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "roleNames": { - "type": "array", - "items": { - "type": "string" - } - }, - "jobTitle": { - "type": "string" - }, - "department": { - "type": "string" - }, - "hierarchy": { - "type": "string" - } - }, - "required": [ - "email" - ] - }, - "User": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "roles": { - "type": "array", - "items": { - "$ref": "#/components/schemas/UserRole" - } - }, - "jobTitle": { - "type": "string" - }, - "department": { - "type": "string" - }, - "hierarchy": { - "type": "string" - }, - "bio": { - "type": "string" - }, - "companyName": { - "type": "string" - }, - "personalSite": { - "type": "string" - }, - "customer": { - "$ref": "#/components/schemas/Customer" - }, - "mfaStatus": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "status": { - "type": "string" - }, - "lastLogin": { - "type": "string" - } - }, - "required": [ - "id", - "name", - "email", - "roles", - "jobTitle", - "department", - "hierarchy", - "bio", - "companyName", - "personalSite", - "mfaStatus", - "createdAt", - "updatedAt", - "status" - ] - }, - "CreateUserRes": { - "type": "object", - "properties": { - "user": { - "$ref": "#/components/schemas/User" - } - }, - "required": [ - "user" - ] - }, - "BatchCreateUserReq": { - "type": "object", - "properties": { - "users": { - "type": "array", - "items": { - "$ref": "#/components/schemas/CreateUserReq" - } - } - }, - "required": [ - "users" - ] - }, - "ErrorUsers": { - "type": "object", - "properties": { - "user": { - "$ref": "#/components/schemas/CreateUserReq" - }, - "error": { - "type": "string" - } - }, - "required": [ - "user", - "error" - ] - }, - "BatchCreateUserRes": { - "type": "object", - "properties": { - "usersCreated": { - "type": "array", - "items": { - "$ref": "#/components/schemas/User" - } - }, - "errorUsers": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ErrorUsers" - } - } - }, - "required": [ - "usersCreated", - "errorUsers" - ] - }, - "DeleteUserRes": { - "type": "object", - "properties": { - "id": { - "type": "string" - } - }, - "required": [ - "id" - ] - }, - "SetUserRolesReq": { - "type": "object", - "properties": { - "userId": { - "type": "string" - }, - "roleNames": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "userId", - "roleNames" - ] - }, - "SetUserRolesRes": { - "type": "object", - "properties": { - "user": { - "$ref": "#/components/schemas/IUserByCustomer" - } - }, - "required": [ - "user" - ] - }, - "AssignRoleToUserReq": { - "type": "object", - "properties": { - "userId": { - "type": "string" - }, - "roleId": { - "type": "string" - } - }, - "required": [ - "userId", - "roleId" - ] - }, - "UpdateUserReq": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "jobTitle": { - "type": "string" - }, - "department": { - "type": "string" - }, - "hierarchy": { - "type": "string" - }, - "bio": { - "type": "string" - }, - "personalSite": { - "type": "string" - }, - "companyName": { - "type": "string" - }, - "roleNames": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "UpdateUserRes": { - "type": "object", - "properties": { - "user": { - "$ref": "#/components/schemas/IUserByCustomer" - } - }, - "required": [ - "user" - ] - }, - "UserNoRolesAndCustomer": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "email": { - "type": "string" - }, - "jobTitle": { - "type": "string" - }, - "department": { - "type": "string" - }, - "hierarchy": { - "type": "string" - }, - "bio": { - "type": "string" - }, - "companyName": { - "type": "string" - }, - "personalSite": { - "type": "string" - }, - "mfaStatus": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "status": { - "type": "string" - }, - "lastLogin": { - "type": "string" - } - }, - "required": [ - "id", - "name", - "email", - "jobTitle", - "department", - "hierarchy", - "bio", - "companyName", - "personalSite", - "mfaStatus", - "createdAt", - "updatedAt", - "status" - ] - }, - "RoleDto": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PermissionDto" - } - }, - "users": { - "type": "array", - "items": { - "$ref": "#/components/schemas/UserNoRolesAndCustomer" - } - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "isPublic": { - "type": "boolean" - }, - "customer": { - "$ref": "#/components/schemas/Customer" - }, - "usage": { - "type": "string" - }, - "canUntoggle": { - "type": "boolean", - "description": "Whether this role can be untoggled" - } - }, - "required": [ - "id", - "name", - "description", - "createdAt", - "updatedAt", - "isPublic", - "customer", - "usage" - ] - }, - "GetRolesByCustomerRes": { - "type": "object", - "properties": { - "roles": { - "type": "array", - "items": { - "$ref": "#/components/schemas/RoleDto" - } - } - }, - "required": [ - "roles" - ] - }, - "CreateRoleReq": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "permissionIds": { - "type": "array", - "items": { - "type": "string" - } - }, - "userIds": { - "type": "array", - "items": { - "type": "string" - } - }, - "description": { - "type": "string" - } - }, - "required": [ - "name", - "permissionIds", - "description" - ] - }, - "CustomerRole": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PermissionDto" - } - }, - "users": { - "type": "array", - "items": { - "$ref": "#/components/schemas/UserNoRolesAndCustomer" - } - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "isPublic": { - "type": "boolean" - }, - "usage": { - "type": "string" - }, - "canUntoggle": { - "type": "boolean", - "description": "Whether this role can be untoggled" - } - }, - "required": [ - "id", - "name", - "description", - "createdAt", - "updatedAt", - "isPublic", - "usage" - ] - }, - "CreateRoleRes": { - "type": "object", - "properties": { - "role": { - "$ref": "#/components/schemas/CustomerRole" - } - }, - "required": [ - "role" - ] - }, - "GrantPermissionsToRoleReq": { - "type": "object", - "properties": { - "roleId": { - "type": "string" - }, - "permissionIds": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "roleId", - "permissionIds" - ] - }, - "GrantPermissionsToRoleRes": { - "type": "object", - "properties": { - "role": { - "$ref": "#/components/schemas/RoleDto" - } - }, - "required": [ - "role" - ] - }, - "RevokePermissionsFromRoleReq": { - "type": "object", - "properties": { - "roleId": { - "type": "string" - }, - "permissionIds": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "roleId", - "permissionIds" - ] - }, - "RevokePermissionsFromRoleRes": { - "type": "object", - "properties": { - "role": { - "$ref": "#/components/schemas/RoleDto" - } - }, - "required": [ - "role" - ] - }, - "SetRolePermissionsReq": { - "type": "object", - "properties": { - "roleId": { - "type": "string" - }, - "permissionIds": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "roleId", - "permissionIds" - ] - }, - "SetRolePermissionsRes": { - "type": "object", - "properties": { - "role": { - "$ref": "#/components/schemas/RoleDto" - } - }, - "required": [ - "role" - ] - }, - "SetRoleUsersReq": { - "type": "object", - "properties": { - "roleId": { - "type": "string" - }, - "userIds": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "roleId", - "userIds" - ] - }, - "SetRoleUsersRes": { - "type": "object", - "properties": { - "role": { - "$ref": "#/components/schemas/RoleDto" - } - }, - "required": [ - "role" - ] - }, - "UpdateRoleReq": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - } - } - }, - "UpdateRoleRes": { - "type": "object", - "properties": { - "role": { - "$ref": "#/components/schemas/RoleDto" - } - }, - "required": [ - "role" - ] - }, - "PermissionGroupPermission": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "id": { - "type": "number" - } - }, - "required": [ - "name", - "id" - ] - }, - "PermissionGroup": { - "type": "object", - "properties": { - "title": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PermissionGroupPermission" - } - } - }, - "required": [ - "title", - "permissions" - ] - }, - "GetPublicPermissionsRes": { - "type": "object", - "properties": { - "permissionGroups": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PermissionGroup" - } - } - }, - "required": [ - "permissionGroups" - ] - }, - "CreateConnectorDto": { - "type": "object", - "properties": { - "file": { - "type": "string", - "format": "binary" - }, - "connector": { - "type": "string" - } - }, - "required": [ - "file", - "connector" - ] - }, - "UpdateDto": { - "type": "object", - "properties": { - "file": { - "type": "string", - "format": "binary" - }, - "version": { - "type": "string" - }, - "category": { - "type": "string" - }, - "plugin": { - "type": "string" - }, - "image": { - "type": "string" - }, - "docs": { - "type": "string" - }, - "name": { - "type": "string" - }, - "options": { - "type": "object", - "description": "skip_select_entities: boolean skip_select_columns: boolean;" - }, - "connection_controls": { - "type": "array", - "items": { - "properties": {} - } - } - }, - "required": [ - "version" - ] - }, - "AddTagDto": { - "type": "object", - "properties": { - "tags": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "tags" - ] - }, - "RemoveTagDto": { - "type": "object", - "properties": { - "tags": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "tags" - ] - }, - "NetworkConfigToCatalogDto": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "customer_id": { - "type": "string" - }, - "plugin": { - "type": "string" - }, - "type": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "network_type_id": { - "type": "string" - }, - "network_type_name": { - "type": "string" - } - }, - "required": [ - "id", - "customer_id", - "plugin", - "type", - "name", - "description", - "created_at", - "updated_at", - "network_type_id", - "network_type_name" - ] - }, - "FindAllNetworkConfigsRes": { - "type": "object", - "properties": { - "network_configs": { - "type": "array", - "items": { - "$ref": "#/components/schemas/NetworkConfigToCatalogDto" - } - }, - "message": { - "type": "string" - } - }, - "required": [ - "network_configs", - "message" - ] - }, - "CreateNetworkConfigReq": { - "type": "object", - "properties": { - "network_type_id": { - "type": "string" - }, - "network_type_name": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "ovpn_file": { - "type": "string", - "format": "binary" - } - }, - "required": [ - "network_type_id", - "network_type_name", - "name", - "description", - "ovpn_file" - ] - }, - "CreateNetworkConfigRes": { - "type": "object", - "properties": { - "network_config": { - "$ref": "#/components/schemas/NetworkConfigToCatalogDto" - } - }, - "required": [ - "network_config" - ] - }, - "AvailableEntity": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "replication_methods": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "name", - "replication_methods" - ] - }, - "GetAvailableEntitiesRes": { - "type": "object", - "properties": { - "entities": { - "type": "array", - "items": { - "$ref": "#/components/schemas/AvailableEntity" - } - } - }, - "required": [ - "entities" - ] - }, - "Column": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string" - } - }, - "required": [ - "name", - "type" - ] - }, - "TableColumns": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "columns": { - "type": "array", - "items": { - "type": "string" - } - }, - "references": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Column" - } - }, - "destination": { - "type": "object" - }, - "type": { - "type": "string" - }, - "identifier_columns": { - "type": "array", - "items": { - "type": "string" - } - }, - "reference_column": { - "$ref": "#/components/schemas/Column" - } - }, - "required": [ - "name", - "columns", - "destination", - "type" - ] - }, - "CreateInputReq": { - "type": "object", - "properties": { - "tables": { - "type": "array", - "items": { - "$ref": "#/components/schemas/TableColumns" - } - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "plugin": { - "type": "string" - }, - "type": { - "type": "string" - } - }, - "required": [ - "tables", - "name", - "description", - "plugin" - ] - }, - "Input": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "tables": { - "type": "array", - "items": { - "$ref": "#/components/schemas/TableColumns" - } - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "plugin": { - "type": "string" - }, - "type": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - }, - "required": [ - "id", - "tables", - "name", - "description", - "plugin", - "created_at", - "updated_at" - ] - }, - "ICreatePipelineV2Req": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "type": { - "type": "string" - }, - "connection_id": { - "type": "string" - }, - "cron": { - "type": "string" - }, - "input_id": { - "type": "string" - }, - "transformations_ids": { - "type": "array", - "items": { - "type": "string" - } - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "properties": { - "type": "object" - }, - "connector_name": { - "type": "string" - }, - "connector_plugin": { - "type": "string" - }, - "connector_version": { - "type": "string" - }, - "image_url": { - "type": "string" - } - }, - "required": [ - "name", - "description", - "type", - "connection_id", - "cron", - "connector_name", - "connector_plugin", - "connector_version", - "image_url" - ] - }, - "IPipelineV2": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "type": { - "type": "string" - }, - "connection_id": { - "type": "string" - }, - "cron": { - "type": "string" - }, - "input_id": { - "type": "string" - }, - "transformations_ids": { - "type": "array", - "items": { - "type": "string" - } - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "properties": { - "type": "object" - }, - "connector_name": { - "type": "string" - }, - "connector_plugin": { - "type": "string" - }, - "connector_version": { - "type": "string" - }, - "image_url": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - } - }, - "required": [ - "id", - "name", - "description", - "type", - "connection_id", - "cron", - "connector_name", - "connector_plugin", - "connector_version", - "image_url", - "created_at", - "updated_at" - ] - }, - "IInitUploadCSVFile": { - "type": "object", - "properties": { - "file_name": { - "type": "string" - }, - "parts": { - "type": "number" - } - }, - "required": [ - "file_name", - "parts" - ] - }, - "ICompleteUploadCSVFile": { - "type": "object", - "properties": { - "upload_id": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "parts": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "upload_id", - "file_name", - "parts" - ] - }, - "ICreatePipelineCSVFile": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "encoding": { - "type": "string" - }, - "sep": { - "type": "string" - }, - "header": { - "type": "boolean" - }, - "engine": { - "type": "string" - } - }, - "required": [ - "name", - "file_name", - "description", - "encoding", - "sep", - "header", - "engine" - ] - }, - "ValidationTableDTO": { - "type": "object", - "properties": { - "tables": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "tables" - ] - }, "EnforceMfa": { "type": "object", "properties": { @@ -10631,595 +8046,6 @@ "links" ] }, - "IRole": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "id": { - "type": "string" - } - }, - "required": [ - "name", - "id" - ] - }, - "IUser": { - "type": "object", - "properties": { - "username": { - "type": "string" - }, - "id": { - "type": "string" - } - }, - "required": [ - "username", - "id" - ] - }, - "IComment": { - "type": "object", - "properties": { - "id": { - "type": "number" - }, - "message": { - "type": "string" - }, - "created_at": { - "type": "string" - }, - "username": { - "type": "string" - } - }, - "required": [ - "id", - "message", - "created_at", - "username" - ] - }, - "EmbedObject": { - "type": "object", - "properties": { - "url": { - "type": "string" - } - }, - "required": [ - "url" - ] - }, - "IDataAsset": { - "type": "object", - "properties": { - "id": { - "type": "number" - }, - "database_name": { - "type": "string" - }, - "table_name": { - "type": "string" - }, - "num_columns": { - "type": "number" - }, - "location": { - "type": "string" - }, - "created_time": { - "type": "string" - }, - "table_type": { - "type": "string" - }, - "size_in_bytes": { - "type": "number" - }, - "num_files": { - "type": "number" - }, - "data_encrypted": { - "type": "string" - }, - "num_rows": { - "type": "number" - }, - "display_name": { - "type": "string" - }, - "pipeline_id": { - "type": "string" - }, - "user_id": { - "type": "string" - }, - "description": { - "type": "string" - }, - "data_asset_type": { - "type": "string" - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "owner": { - "type": "string" - }, - "roles": { - "type": "array", - "items": { - "$ref": "#/components/schemas/IRole" - } - }, - "users": { - "type": "array", - "items": { - "$ref": "#/components/schemas/IUser" - } - }, - "dashboard_id": { - "type": "number" - }, - "name": { - "type": "string" - }, - "parameters": { - "type": "array", - "items": { - "type": "string" - } - }, - "embedding_params": { - "type": "object" - }, - "created_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "comments": { - "type": "array", - "items": { - "$ref": "#/components/schemas/IComment" - } - }, - "embed": { - "$ref": "#/components/schemas/EmbedObject" - }, - "share_type": { - "type": "string", - "enum": [ - "none", - "public", - "private" - ] - }, - "docs": { - "type": "string" - } - }, - "required": [ - "id", - "display_name", - "pipeline_id", - "user_id", - "description", - "data_asset_type", - "tags", - "owner", - "roles", - "users", - "created_at", - "comments" - ] - }, - "ICatalogAllResponse": { - "type": "object", - "properties": { - "data_assets": { - "type": "array", - "items": { - "$ref": "#/components/schemas/IDataAsset" - } - }, - "total": { - "type": "number" - } - }, - "required": [ - "data_assets", - "total" - ] - }, - "IColumnMetadata": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "table_name": { - "type": "string" - }, - "column_name": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "missing_count": { - "type": "string" - }, - "min": { - "type": "string" - }, - "max": { - "type": "string" - }, - "count": { - "type": "string" - }, - "histogram": { - "type": "string" - }, - "mean": { - "type": "string" - }, - "median": { - "type": "string" - }, - "missing_rate": { - "type": "string" - }, - "stddev": { - "type": "string" - }, - "unique_count": { - "type": "string" - }, - "unique_rate": { - "type": "string" - }, - "analyzer_rules": { - "type": "array", - "items": { - "type": "string" - } - }, - "version": { - "type": "string" - } - }, - "required": [ - "id", - "table_name", - "column_name", - "data_type", - "missing_count", - "min", - "max", - "count", - "histogram", - "mean", - "median", - "missing_rate", - "stddev", - "unique_count", - "unique_rate", - "analyzer_rules", - "version" - ] - }, - "IColumnsMetadataResponse": { - "type": "object", - "properties": { - "columns_metadata": { - "type": "array", - "items": { - "$ref": "#/components/schemas/IColumnMetadata" - } - } - }, - "required": [ - "columns_metadata" - ] - }, - "IData": { - "type": "object", - "properties": { - "var": { - "type": "number" - }, - "date": { - "type": "string" - }, - "last": { - "type": "number" - }, - "maximum": { - "type": "number" - }, - "minimum": { - "type": "number" - }, - "day_opening": { - "type": "number" - } - }, - "required": [ - "var", - "date", - "last", - "maximum", - "minimum", - "day_opening" - ] - }, - "IPreview": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "database_name": { - "type": "string" - }, - "table_name": { - "type": "string" - }, - "data": { - "type": "array", - "items": { - "$ref": "#/components/schemas/IData" - } - } - }, - "required": [ - "id", - "database_name", - "table_name", - "data" - ] - }, - "IPreviewResponse": { - "type": "object", - "properties": { - "preview": { - "$ref": "#/components/schemas/IPreview" - } - }, - "required": [ - "preview" - ] - }, - "IDocs": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "docs": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "table_id": { - "type": "string" - } - }, - "required": [ - "id", - "docs", - "updated_at", - "table_id" - ] - }, - "IDocsResponse": { - "type": "object", - "properties": { - "docs": { - "$ref": "#/components/schemas/IDocs" - } - }, - "required": [ - "docs" - ] - }, - "IUpdateDataRequest": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "embed": { - "$ref": "#/components/schemas/EmbedObject" - }, - "share_type": { - "type": "string", - "enum": [ - "none", - "public", - "private" - ] - }, - "docs": { - "type": "string" - } - }, - "required": [ - "name", - "description", - "tags" - ] - }, - "IOneDataAsset": { - "type": "object", - "properties": { - "data_asset": { - "$ref": "#/components/schemas/IDataAsset" - } - }, - "required": [ - "data_asset" - ] - }, - "ICreateDataAsset": { - "type": "object", - "properties": { - "display_name": { - "type": "string" - }, - "description": { - "type": "string" - }, - "data_asset_type": { - "type": "string" - }, - "external_url": { - "type": "string" - }, - "location": { - "type": "string" - }, - "embed": { - "$ref": "#/components/schemas/EmbedObject" - }, - "docs": { - "type": "string" - } - }, - "required": [ - "display_name", - "description", - "data_asset_type", - "external_url", - "location" - ] - }, - "IMakeAComment": { - "type": "object", - "properties": { - "message": { - "type": "string" - } - }, - "required": [ - "message" - ] - }, - "CommentToDelete": { - "type": "object", - "properties": { - "id": { - "type": "number" - }, - "username": { - "type": "string" - } - }, - "required": [ - "id", - "username" - ] - }, - "IDeleteComment": { - "type": "object", - "properties": { - "comment": { - "$ref": "#/components/schemas/CommentToDelete" - } - }, - "required": [ - "comment" - ] - }, - "TriggerCatalogReq": { - "type": "object", - "properties": { - "table_name": { - "type": "string", - "description": "Name of the table on Snowflake", - "example": "TB__4DXSD4_TEST" - }, - "table_schema": { - "type": "string", - "description": "Schema where the asset is located. If not set defaults to \"PUBLIC\"" - }, - "pipeline_id": { - "type": "string", - "description": "Id of the pipeline that generated the asset." - } - }, - "required": [ - "table_name" - ] - }, - "TriggerCatalogRes": { - "type": "object", - "properties": { - "session": { - "type": "string" - } - }, - "required": [ - "session" - ] - }, - "GetDatasetCatalogTaskRes": { - "type": "object", - "properties": { - "created_at": { - "type": "string" - }, - "customer_name": { - "type": "string" - }, - "session_id": { - "type": "string" - }, - "status": { - "type": "string" - }, - "table_name": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "updated_by": { - "type": "string" - } - }, - "required": [ - "created_at", - "customer_name", - "session_id", - "status", - "table_name", - "updated_at", - "updated_by" - ] - }, "CreateShareMetadataDto": { "type": "object", "properties": { @@ -11703,4 +8529,3 @@ } } } -} \ No newline at end of file diff --git a/package.json b/package.json index fd480de..586d74c 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.40.0-beta.2", + "@dadosfera/protospack-v2": "^3.40.0-beta.3", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/pipelinesV2/pipelines.controller.ts b/src/modules/pipelinesV2/pipelines.controller.ts index 1c00bcd..ab39d38 100644 --- a/src/modules/pipelinesV2/pipelines.controller.ts +++ b/src/modules/pipelinesV2/pipelines.controller.ts @@ -224,11 +224,12 @@ export class PipelinesController { const result = await this.pipelinesClientService .findOne({ id }, metadata) - .then((res) => { + .then(async (res) => { //{pipeline:{tables: {tables: [], input_id: ''}}} - let tables = JSON.parse(res.pipeline.config.tables); - const input_id = tables?.input_id; - if (tables?.tables) tables = tables.tables; + const parsed = JSON.parse(res.pipeline.config.tables); + const input_id = parsed?.input_id; + const tables = parsed?.tables ?? parsed; + Object.assign(res.pipeline, { transformations: res.pipeline.transformations ? JSON.parse(res.pipeline.transformations) diff --git a/src/modules/platform-api/platform-api.controller.ts b/src/modules/platform-api/platform-api.controller.ts index 8c42102..74caaf9 100644 --- a/src/modules/platform-api/platform-api.controller.ts +++ b/src/modules/platform-api/platform-api.controller.ts @@ -962,6 +962,80 @@ export class PlatformApiController { ); } + @Delete('jobs/:jobId') + @ApiOperation({ summary: 'Delete a job and sync pipeline data' }) + @RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE) + async deleteJob( + @Param('jobId') jobId: string, + @User() user: RequestUser, + ) { + // Get job details before deletion to extract table_name and connector type + const normalizedJobId = this.normalizeJobId(jobId); + let tableName: string | undefined; + + this.logger.info('[deleteJob] Start', { jobId }); + + try { + const job = await this.platformApiService.proxy( + 'GET', + `/jobs/${normalizedJobId}`, + user, + ); + tableName = job?.source_config?.table_name; + this.logger.info('[deleteJob] Job fetched', { jobId, tableName }); + } catch (err) { + this.logger.warn('[deleteJob] Could not fetch job before deletion', { jobId, error: err.message }); + } + + const result = await this.platformApiService.proxy( + 'DELETE', + `/jobs/${normalizedJobId}`, + user, + ); + this.logger.info('[deleteJob] Job deleted from platform-api', { jobId }); + + // Mark the table as deleted via in-factory (which has the correct DynamoDB credentials). + // config.tables in ES stores the DynamoDB input ID (legacy naming). + if (tableName) { + try { + const esPipelineId = this.extractPipelineIdFromJobId(jobId); + const esPipeline = await this.elasticsearchService.getPipeline( + user.customer_name, + esPipelineId, + ); + // config.tables holds the DynamoDB input ID (see syncJobInputToDynamoDB for context) + const dynamoInputId = esPipeline?.config?.tables; + + if (dynamoInputId) { + const info = { customer_id: user.customer_id, user_id: user.user_id, customer: user.customer_name }; + + // Read current input to get full tables array (adjustInputPayload flattens the response) + const { input } = await this.inputsService.findOne({ id: dynamoInputId, info }); + const currentTables: any[] = input?.tables ?? []; + + const updatedTables = currentTables.map((t: any) => + t.name === tableName ? { ...t, status: 'deleted' } : t, + ); + + await this.inputsService.update(dynamoInputId, { tables: updatedTables }, info); + this.logger.info('[deleteJob] Marked table as deleted via in-factory', { jobId, tableName }); + } else { + this.logger.warn('[deleteJob] No input ID in ES pipeline, skipping DynamoDB update', { jobId, tableName }); + } + } catch (error) { + this.logger.error('[deleteJob] Failed to mark table as deleted via in-factory', { + jobId, + tableName, + error: error.message, + }); + } + } else { + this.logger.warn('[deleteJob] tableName not resolved, skipping DynamoDB update', { jobId }); + } + + return result; + } + // ==================== JOBS - JDBC SYNC MODE ROUTES ==================== @Get('jobs/jdbc/:jobId') diff --git a/src/modules/platform-api/platform-api.module.ts b/src/modules/platform-api/platform-api.module.ts index b78e680..d202700 100644 --- a/src/modules/platform-api/platform-api.module.ts +++ b/src/modules/platform-api/platform-api.module.ts @@ -8,9 +8,10 @@ import { ElasticsearchModule } from '../../services/elasticsearch'; import { DynamoDBModule } from '../../services/dynamodb'; import { CustomersModule } from '../customers/customers.module'; import { CatalogModule } from '../catalog/catalog.module'; +import { PipelinesV2Module } from '../pipelinesV2/pipelines.module'; @Module({ - imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule], + imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, PipelinesV2Module], controllers: [PlatformApiController], providers: [PlatformApiService, DadosferaLogger], exports: [PlatformApiService], diff --git a/src/services/dynamodb/dynamodb.service.ts b/src/services/dynamodb/dynamodb.service.ts index 15dde41..a0d3537 100644 --- a/src/services/dynamodb/dynamodb.service.ts +++ b/src/services/dynamodb/dynamodb.service.ts @@ -30,6 +30,7 @@ export interface InputDocument { type: string; columns?: string[]; reference_column?: ReferenceColumn; + status?: string; }>; credentials?: Record; } @@ -125,9 +126,13 @@ export class DynamoDBService { }, }); - const { Item } = await this.documentClient.send(getCommand); - - return Item as InputDocument | null; + try { + const { Item } = await this.documentClient.send(getCommand); + return Item as InputDocument | null; + } catch (error) { + this.logger.error('DynamoDB: findInput failed', { inputId, clientId, error: error.message }); + throw error; + } } async deleteInput(clientId: string, inputId: string): Promise { @@ -163,6 +168,7 @@ export class DynamoDBService { type?: string; columns?: string[]; reference_column?: ReferenceColumn | null; + status?: string; }, ): Promise { const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); @@ -203,6 +209,7 @@ export class DynamoDBService { updatedTable.reference_column = changes.reference_column; } } + if ('status' in changes) updatedTable.status = changes.status; tables[tableIndex] = updatedTable; @@ -231,4 +238,53 @@ export class DynamoDBService { throw error; } } + + /** + * Remove a specific table entry from the input document. + * Fetches the current document, removes the matching table, and saves. + */ + async removeInputTable( + clientId: string, + inputId: string, + tableName: string, + ): Promise { + const dynamoTableName = DYNAMODB_CONFIG.inputsTable(); + + this.logger.info('DynamoDB: Removing input table', { + inputId, + tableName, + }); + + const current = await this.findInput(clientId, inputId); + if (!current) { + this.logger.warn('DynamoDB: Input not found for table removal', { inputId }); + return; + } + + const tables = (current.tables || []).filter((t) => t.name !== tableName); + + const putCommand = new PutCommand({ + TableName: dynamoTableName, + Item: { + ...current, + tables, + updated_at: new Date().toISOString(), + }, + }); + + try { + await this.documentClient.send(putCommand); + this.logger.info('DynamoDB: Input table removed successfully', { + inputId, + tableName, + }); + } catch (error) { + this.logger.error('DynamoDB: Failed to remove input table', { + inputId, + tableName, + error: error.message, + }); + throw error; + } + } }