mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-07 23:39:08 +00:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
901518d26c | ||
|
|
4ab8147149 | ||
|
|
0086528d51 | ||
|
|
bf12598915 | ||
|
|
efb0b58648 | ||
|
|
8a9b58b612 | ||
|
|
837a9d7265 | ||
|
|
d0122a9c20 | ||
|
|
968f75b688 | ||
|
|
877cb9d281 | ||
|
|
59efdb6272 | ||
|
|
7b1049224c | ||
|
|
0369f10b5c | ||
|
|
ecca9106f0 | ||
|
|
efc1f49d92 | ||
|
|
2d46ac3213 | ||
|
|
271174176b |
+2
-2
@@ -22,7 +22,7 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
|||||||
# run aws cli without mounting secret, because CI already has AWS credentials
|
# run aws cli without mounting secret, because CI already has AWS credentials
|
||||||
FROM build_base AS ci_image
|
FROM build_base AS ci_image
|
||||||
RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
||||||
RUN npm ci
|
RUN npm ci --ignore-scripts
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|
||||||
@@ -37,7 +37,7 @@ FROM build_base AS dev
|
|||||||
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
|
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
|
||||||
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
||||||
# flag --build-from-source is required to force-build sqlite3
|
# flag --build-from-source is required to force-build sqlite3
|
||||||
RUN npm ci
|
RUN npm ci --ignore-scripts
|
||||||
COPY . .
|
COPY . .
|
||||||
ENTRYPOINT npm run start:dev
|
ENTRYPOINT npm run start:dev
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -22,7 +22,7 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
|||||||
FROM build_base AS build
|
FROM build_base AS build
|
||||||
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
|
RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
|
||||||
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
|
||||||
RUN npm ci
|
RUN npm ci --ignore-scripts
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN npm run build
|
RUN npm run build
|
||||||
|
|
||||||
|
|||||||
@@ -4705,6 +4705,43 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/platform/pipelines/{pipelineId}/inputs/{inputId}": {
|
||||||
|
"delete": {
|
||||||
|
"operationId": "PlatformApiController_deleteTable",
|
||||||
|
"summary": "Mark a table as deleted and delete its associated job via platform-api",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"name": "pipelineId",
|
||||||
|
"required": true,
|
||||||
|
"in": "path",
|
||||||
|
"schema": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "inputId",
|
||||||
|
"required": true,
|
||||||
|
"in": "path",
|
||||||
|
"schema": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"description": ""
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tags": [
|
||||||
|
"Platform API"
|
||||||
|
],
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"access-token": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/platform/jobs/jdbc/{jobId}": {
|
"/platform/jobs/jdbc/{jobId}": {
|
||||||
"get": {
|
"get": {
|
||||||
"operationId": "PlatformApiController_getJdbcJob",
|
"operationId": "PlatformApiController_getJdbcJob",
|
||||||
|
|||||||
Generated
+8
-9
@@ -17,7 +17,7 @@
|
|||||||
"@aws-sdk/signature-v4": "^3.370.0",
|
"@aws-sdk/signature-v4": "^3.370.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack": "2.5.3",
|
"@dadosfera/protospack": "2.5.3",
|
||||||
"@dadosfera/protospack-v2": "3.40.0-beta.2",
|
"@dadosfera/protospack-v2": "^3.40.0-beta.5",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
@@ -31,7 +31,7 @@
|
|||||||
"@nestjs/schematics": "^9.2.0",
|
"@nestjs/schematics": "^9.2.0",
|
||||||
"@nestjs/swagger": "^6.3.0",
|
"@nestjs/swagger": "^6.3.0",
|
||||||
"@nestjs/testing": "^9.4.3",
|
"@nestjs/testing": "^9.4.3",
|
||||||
"axios": "^0.30.2",
|
"axios": "0.30.3",
|
||||||
"cache-manager": "^5.1.4",
|
"cache-manager": "^5.1.4",
|
||||||
"cache-manager-ioredis-yet": "^1.1.0",
|
"cache-manager-ioredis-yet": "^1.1.0",
|
||||||
"class-transformer": "^0.5.1",
|
"class-transformer": "^0.5.1",
|
||||||
@@ -1745,10 +1745,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@dadosfera/protospack-v2": {
|
"node_modules/@dadosfera/protospack-v2": {
|
||||||
"version": "3.40.0-beta.2",
|
"version": "3.40.0-beta.5",
|
||||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.2.tgz",
|
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.5.tgz",
|
||||||
"integrity": "sha512-WMoL9OhKJ05qDojd7cPcP7x45bHHnnQttfVoTWrc+lOkcVHygu0ob52ArfD8RRmpAXezBh9vDSbxz45n/lxbKg==",
|
"integrity": "sha512-iocKv/XXp2jKAasO5ONgm31cKfLgNsU4pEKZMr6YnR7nQaH11WcW7rnuagNxWoik++wLUqbYyf0bZWRDzMlCPA==",
|
||||||
"license": "ISC",
|
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"rxjs": "^7.5.5"
|
"rxjs": "^7.5.5"
|
||||||
@@ -5599,9 +5598,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/axios": {
|
"node_modules/axios": {
|
||||||
"version": "0.30.2",
|
"version": "0.30.3",
|
||||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.30.2.tgz",
|
"resolved": "https://registry.npmjs.org/axios/-/axios-0.30.3.tgz",
|
||||||
"integrity": "sha512-0pE4RQ4UQi1jKY6p7u6i1Tkzqmu+d+/tHS7Q7rKunWLB9WyilBTpHHpXzPNMDj5hTbK0B0PTLSz07yqMBiF6xg==",
|
"integrity": "sha512-5/tmEb6TmE/ax3mdXBc/Mi6YdPGxQsv+0p5YlciXWt3PHIn0VamqCXhRMtScnwY3lbgSXLneOuXAKUhgmSRpwg==",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"follow-redirects": "^1.15.4",
|
"follow-redirects": "^1.15.4",
|
||||||
|
|||||||
+6
-2
@@ -35,7 +35,7 @@
|
|||||||
"@aws-sdk/signature-v4": "^3.370.0",
|
"@aws-sdk/signature-v4": "^3.370.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack": "2.5.3",
|
"@dadosfera/protospack": "2.5.3",
|
||||||
"@dadosfera/protospack-v2": "3.40.0-beta.2",
|
"@dadosfera/protospack-v2": "^3.40.0-beta.5",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
@@ -49,7 +49,7 @@
|
|||||||
"@nestjs/schematics": "^9.2.0",
|
"@nestjs/schematics": "^9.2.0",
|
||||||
"@nestjs/swagger": "^6.3.0",
|
"@nestjs/swagger": "^6.3.0",
|
||||||
"@nestjs/testing": "^9.4.3",
|
"@nestjs/testing": "^9.4.3",
|
||||||
"axios": "^0.30.2",
|
"axios": "0.30.3",
|
||||||
"cache-manager": "^5.1.4",
|
"cache-manager": "^5.1.4",
|
||||||
"cache-manager-ioredis-yet": "^1.1.0",
|
"cache-manager-ioredis-yet": "^1.1.0",
|
||||||
"class-transformer": "^0.5.1",
|
"class-transformer": "^0.5.1",
|
||||||
@@ -81,6 +81,7 @@
|
|||||||
"swagger-ui-express": "^4.6.3"
|
"swagger-ui-express": "^4.6.3"
|
||||||
},
|
},
|
||||||
"overrides": {
|
"overrides": {
|
||||||
|
"axios": "0.30.3",
|
||||||
"form-data": "^4.0.4",
|
"form-data": "^4.0.4",
|
||||||
"body-parser": "^1.20.3",
|
"body-parser": "^1.20.3",
|
||||||
"cross-spawn": "^7.0.5",
|
"cross-spawn": "^7.0.5",
|
||||||
@@ -117,5 +118,8 @@
|
|||||||
"ts-node": "^10.9.1",
|
"ts-node": "^10.9.1",
|
||||||
"tsconfig-paths": "^3.14.2",
|
"tsconfig-paths": "^3.14.2",
|
||||||
"typescript": "^4.9.5"
|
"typescript": "^4.9.5"
|
||||||
|
},
|
||||||
|
"resolutions": {
|
||||||
|
"axios": "0.30.3"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -288,4 +288,12 @@ export class InputsService {
|
|||||||
};
|
};
|
||||||
return formatedPayload;
|
return formatedPayload;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async markTableDeleted(data: { input_id: string; table_name: string; info: Info }) {
|
||||||
|
return lastValueFrom(this.inputWriteService.MarkTableDeleted(data));
|
||||||
|
}
|
||||||
|
|
||||||
|
async unmarkTableDeleted(data: { input_id: string; table_name: string; info: Info }) {
|
||||||
|
return lastValueFrom((this.inputWriteService as any).UnmarkTableDeleted(data));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,6 +53,9 @@ import { TableColumns } from '../inputs/dtos/input.model';
|
|||||||
import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
|
import { UpdateInputRequest } from '../inputs/dtos/old_interfaces';
|
||||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||||
|
|
||||||
|
type PipelineTable = { name: string; job_id?: string; is_deleted?: boolean; [key: string]: any };
|
||||||
|
type PipelineTablesConfig = { input_id?: string; tables: PipelineTable[] };
|
||||||
|
|
||||||
@ApiTags('PipelinesV2')
|
@ApiTags('PipelinesV2')
|
||||||
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
|
||||||
@UseFilters(new GrpcToHttpExceptionFilter())
|
@UseFilters(new GrpcToHttpExceptionFilter())
|
||||||
@@ -62,6 +65,7 @@ export class PipelinesController {
|
|||||||
constructor(
|
constructor(
|
||||||
@Inject(DadosferaLogger)
|
@Inject(DadosferaLogger)
|
||||||
dadosferaLogger: DadosferaLogger,
|
dadosferaLogger: DadosferaLogger,
|
||||||
|
|
||||||
private pipelinesClientService: PipelinesService,
|
private pipelinesClientService: PipelinesService,
|
||||||
private oldPipelinesService: OldPipelineService,
|
private oldPipelinesService: OldPipelineService,
|
||||||
) {
|
) {
|
||||||
@@ -222,30 +226,27 @@ export class PipelinesController {
|
|||||||
language,
|
language,
|
||||||
});
|
});
|
||||||
|
|
||||||
const result = await this.pipelinesClientService
|
const pipelineRes = await this.pipelinesClientService.findOne({ id }, metadata);
|
||||||
.findOne({ id }, metadata)
|
|
||||||
.then((res) => {
|
|
||||||
//{pipeline:{tables: {tables: [], input_id: ''}}}
|
|
||||||
let tables = JSON.parse(res.pipeline.config.tables);
|
|
||||||
const input_id = tables?.input_id;
|
|
||||||
if (tables?.tables) tables = tables.tables;
|
|
||||||
Object.assign(res.pipeline, {
|
|
||||||
transformations: res.pipeline.transformations
|
|
||||||
? JSON.parse(res.pipeline.transformations)
|
|
||||||
: [],
|
|
||||||
config: {
|
|
||||||
cron: res.pipeline.config.cron,
|
|
||||||
tables,
|
|
||||||
input_id
|
|
||||||
},
|
|
||||||
properties: res.pipeline.properties
|
|
||||||
? JSON.parse(res.pipeline.properties)
|
|
||||||
: {},
|
|
||||||
});
|
|
||||||
return res;
|
|
||||||
});
|
|
||||||
|
|
||||||
return result;
|
const parsed: PipelineTablesConfig = JSON.parse(pipelineRes.pipeline.config.tables);
|
||||||
|
const input_id = parsed.input_id;
|
||||||
|
const tables: PipelineTable[] = parsed.tables ?? [];
|
||||||
|
|
||||||
|
Object.assign(pipelineRes.pipeline, {
|
||||||
|
transformations: pipelineRes.pipeline.transformations
|
||||||
|
? JSON.parse(pipelineRes.pipeline.transformations)
|
||||||
|
: [],
|
||||||
|
config: {
|
||||||
|
cron: pipelineRes.pipeline.config.cron,
|
||||||
|
tables,
|
||||||
|
input_id,
|
||||||
|
},
|
||||||
|
properties: pipelineRes.pipeline.properties
|
||||||
|
? JSON.parse(pipelineRes.pipeline.properties)
|
||||||
|
: {},
|
||||||
|
});
|
||||||
|
|
||||||
|
return pipelineRes;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Patch('/:id')
|
@Patch('/:id')
|
||||||
|
|||||||
@@ -422,6 +422,7 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
try {
|
try {
|
||||||
await this.updatePlatformJobs(
|
await this.updatePlatformJobs(
|
||||||
pipelineIdFormat,
|
pipelineIdFormat,
|
||||||
|
updateInputResponse.input.type,
|
||||||
updateInputDTO,
|
updateInputDTO,
|
||||||
user
|
user
|
||||||
);
|
);
|
||||||
@@ -487,7 +488,7 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async updatePlatformJobs(pipelineId: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
|
async updatePlatformJobs(pipelineId: string, pipelineType: string, updateInputDTO: UpdatePlatformInputRequest, user: RequestUser) {
|
||||||
const jobsUpdated = [];
|
const jobsUpdated = [];
|
||||||
|
|
||||||
for (const [index, table] of updateInputDTO.tables.entries()) {
|
for (const [index, table] of updateInputDTO.tables.entries()) {
|
||||||
@@ -528,8 +529,8 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
|
|
||||||
if (table.type) {
|
if (table.type) {
|
||||||
hasUpdateSyncMode = true;
|
hasUpdateSyncMode = true;
|
||||||
|
|
||||||
jobSyncMode['target_load_type'] = table.type;
|
jobSyncMode['target_load_type'] = table.type;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if(hasUpdateSyncMode) {
|
if(hasUpdateSyncMode) {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import {
|
|||||||
Inject,
|
Inject,
|
||||||
BadRequestException,
|
BadRequestException,
|
||||||
HttpException,
|
HttpException,
|
||||||
|
NotFoundException,
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import { ApiTags, ApiOperation } from '@nestjs/swagger';
|
import { ApiTags, ApiOperation } from '@nestjs/swagger';
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
@@ -29,6 +30,7 @@ import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
|
|||||||
import { CatalogService } from '../catalog/catalog.service';
|
import { CatalogService } from '../catalog/catalog.service';
|
||||||
import { PackTheMetadata } from '../../utils/PackTheMetadata';
|
import { PackTheMetadata } from '../../utils/PackTheMetadata';
|
||||||
import { ValidationTableDTO } from './platform-api.dto';
|
import { ValidationTableDTO } from './platform-api.dto';
|
||||||
|
import { InputsService } from '../inputs/inputs.service';
|
||||||
|
|
||||||
|
|
||||||
type ValidateTablesDTO = {
|
type ValidateTablesDTO = {
|
||||||
@@ -54,6 +56,7 @@ export class PlatformApiController {
|
|||||||
private readonly dynamoDBService: DynamoDBService,
|
private readonly dynamoDBService: DynamoDBService,
|
||||||
private readonly customersService: CustomersService,
|
private readonly customersService: CustomersService,
|
||||||
private readonly catalogService: CatalogService,
|
private readonly catalogService: CatalogService,
|
||||||
|
private readonly inputsService: InputsService,
|
||||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||||
) {
|
) {
|
||||||
this.logger = dadosferaLogger.logger;
|
this.logger = dadosferaLogger.logger;
|
||||||
@@ -962,226 +965,52 @@ export class PlatformApiController {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
|
@Delete('pipelines/:pipelineId/inputs/:inputId')
|
||||||
|
@ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' })
|
||||||
@Get('jobs/jdbc/:jobId')
|
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
||||||
@ApiOperation({ summary: 'Get JDBC job details' })
|
async deleteTable(
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
@Param('pipelineId') pipelineId: string,
|
||||||
async getJdbcJob(@Param('jobId') jobId: string, @User() user: RequestUser) {
|
@Param('inputId') inputId: string,
|
||||||
// Normalize job ID for Platform API (replace - with _)
|
@Body() body: { table_name: string },
|
||||||
const normalizedJobId = this.normalizeJobId(jobId);
|
|
||||||
return this.platformApiService.proxy('GET', `/jobs/jdbc/${normalizedJobId}`, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post('jobs/jdbc/:jobId/sync-mode')
|
|
||||||
@ApiOperation({ summary: 'Update JDBC job sync mode' })
|
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
|
||||||
async updateJdbcSyncMode(
|
|
||||||
@Param('jobId') jobId: string,
|
|
||||||
@Body() body: any,
|
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
) {
|
) {
|
||||||
// Normalize job ID for Platform API (replace - with _)
|
const tableName = body.table_name;
|
||||||
const normalizedJobId = this.normalizeJobId(jobId);
|
const info = {
|
||||||
|
customer_id: user.customer_id,
|
||||||
|
customer: user.customer_name,
|
||||||
|
user_id: user.user_id,
|
||||||
|
};
|
||||||
|
|
||||||
const result = await this.platformApiService.proxy(
|
this.logger.info('deleteTable: marking table as deleted', { inputId, tableName });
|
||||||
'POST',
|
const updatedInput: any = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info });
|
||||||
`/jobs/jdbc/${normalizedJobId}/sync-mode`,
|
this.logger.info('deleteTable: table marked as deleted', { inputId, tableName });
|
||||||
user,
|
|
||||||
body,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Sync to DynamoDB (pass raw jobId for pipeline extraction)
|
|
||||||
await this.syncJdbcSyncModeToDynamoDB(jobId, body, user);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post('jobs/:jobId/rename-tables')
|
|
||||||
@ApiOperation({ summary: 'Rename job output tables and sync to catalog' })
|
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
|
||||||
async renameJobTables(
|
|
||||||
@Param('jobId') jobId: string,
|
|
||||||
@Body() body: RenameTablesBody,
|
|
||||||
@User() user: RequestUser,
|
|
||||||
) {
|
|
||||||
const normalizedJobId = this.normalizeJobId(jobId);
|
|
||||||
|
|
||||||
const currentJob = await this.getJobByAnyConnectorType(normalizedJobId, user);
|
|
||||||
|
|
||||||
const result = await this.platformApiService.proxy(
|
|
||||||
'POST',
|
|
||||||
`/jobs/${normalizedJobId}/rename-tables`,
|
|
||||||
user,
|
|
||||||
body,
|
|
||||||
);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this.syncTableRenameToCatalog(jobId, body, currentJob, user);
|
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||||
|
this.logger.info('deleteTable: fetching pipeline from platform-api', { pipelineId, normalizedPipelineId });
|
||||||
|
const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
|
||||||
|
this.logger.info('deleteTable: pipeline fetched', { jobCount: platformPipeline?.jobs?.length });
|
||||||
|
|
||||||
|
const job = platformPipeline?.jobs?.find((j: any) => j.input?.table_name === tableName);
|
||||||
|
if (!job) throw new NotFoundException(`Job for table '${tableName}' not found in pipeline`);
|
||||||
|
|
||||||
|
this.logger.info('deleteTable: deleting job from platform-api', { jobId: job.job_id });
|
||||||
|
await this.platformApiService.proxy('DELETE', `/jobs/${job.job_id}`, user);
|
||||||
|
this.logger.info('deleteTable: job deleted', { jobId: job.job_id });
|
||||||
|
|
||||||
|
return { name: tableName, is_deleted: updatedInput.is_deleted ?? true, deleted_at: updatedInput.deleted_at };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error('Catalog sync failed, rolling back Snowflake rename', { jobId, error: error.message });
|
this.logger.error('deleteTable: platform-api delete failed, attempting rollback', { tableName, error: error.message });
|
||||||
|
|
||||||
const reverseBody = this.buildSnowflakeRollbackBody(body, currentJob.output_config || {});
|
|
||||||
if (reverseBody) {
|
|
||||||
try {
|
|
||||||
await this.platformApiService.proxy('POST', `/jobs/${normalizedJobId}/rename-tables`, user, reverseBody);
|
|
||||||
this.logger.info('Snowflake rename rolled back', { jobId });
|
|
||||||
} catch (rollbackError) {
|
|
||||||
this.logger.error('Snowflake rollback failed', { jobId, error: rollbackError.message });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new HttpException('Table rename failed: catalog sync error, Snowflake reverted', 500);
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private buildSnowflakeRollbackBody(
|
|
||||||
body: RenameTablesBody,
|
|
||||||
outputConfig: any,
|
|
||||||
): RenameTablesBody | null {
|
|
||||||
const reverse: RenameTablesBody = {};
|
|
||||||
|
|
||||||
if (body.raw) {
|
|
||||||
const nested = outputConfig.raw;
|
|
||||||
const oldTableName = nested?.table_name || outputConfig.table_name;
|
|
||||||
const oldTableSchema = nested?.table_schema || 'PUBLIC';
|
|
||||||
if (oldTableName) reverse.raw = { table_name: oldTableName, table_schema: oldTableSchema };
|
|
||||||
}
|
|
||||||
|
|
||||||
if (body.qualify) {
|
|
||||||
const nested = outputConfig.qualify;
|
|
||||||
if (nested?.table_name) reverse.qualify = { table_name: nested.table_name, table_schema: nested.table_schema || 'STAGED' };
|
|
||||||
}
|
|
||||||
|
|
||||||
return Object.keys(reverse).length > 0 ? reverse : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sync table rename to Elasticsearch and Nimbus.
|
|
||||||
*
|
|
||||||
* For each target (raw, qualify):
|
|
||||||
* 1. Resolve old table name from output_config
|
|
||||||
* 2. Find the ES data asset by pipeline + table + schema
|
|
||||||
* 3. Update ES, Nimbus table-metadata, column-metadata, and data-preview
|
|
||||||
* 4. If any step fails, rollback all completed steps for that target
|
|
||||||
*/
|
|
||||||
private async syncTableRenameToCatalog(
|
|
||||||
jobId: string,
|
|
||||||
body: RenameTablesBody,
|
|
||||||
currentJob: any,
|
|
||||||
user: RequestUser,
|
|
||||||
): Promise<void> {
|
|
||||||
const pipelineId = this.extractPipelineIdFromJobId(jobId);
|
|
||||||
const outputConfig = currentJob.output_config || {};
|
|
||||||
const nimbusUrl = this.catalogService._getNimbusUrl({ info: { customer: user.customer_name } });
|
|
||||||
const databaseName = `DADOSFERA_PRD_${user.customer_name.toUpperCase()}`;
|
|
||||||
|
|
||||||
const targets = this.buildRenameTargets(body, outputConfig);
|
|
||||||
|
|
||||||
for (const { key, oldTableName, oldTableSchema, newValues } of targets) {
|
|
||||||
const rollbackSteps: Array<() => Promise<void>> = [];
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const dataAsset = await this.elasticsearchService.findDataAssetByTable(
|
await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: tableName, info });
|
||||||
user.customer_name, oldTableName, oldTableSchema,
|
} catch (rollbackError) {
|
||||||
);
|
this.logger.error('deleteTable: rollback failed', { tableName, error: rollbackError.message });
|
||||||
|
|
||||||
if (!dataAsset) {
|
|
||||||
this.logger.warn(`No data asset found for ${key}`, { jobId, pipelineId, oldTableName, oldTableSchema });
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const { _es_id: esAssetId, nimbus_id: nimbusId } = dataAsset;
|
|
||||||
const oldValues = { table_name: oldTableName, table_schema: oldTableSchema };
|
|
||||||
|
|
||||||
// ES update
|
|
||||||
const esFields = { name: newValues.table_name, table_name: newValues.table_name, table_schema: newValues.table_schema, display_name: newValues.table_name };
|
|
||||||
await this.elasticsearchService.updateDataAsset(user.customer_name, esAssetId, esFields);
|
|
||||||
rollbackSteps.push(() => this.elasticsearchService.updateDataAsset(
|
|
||||||
user.customer_name, esAssetId,
|
|
||||||
{ name: oldTableName, table_name: oldTableName, table_schema: oldTableSchema, display_name: oldTableName },
|
|
||||||
));
|
|
||||||
|
|
||||||
const newTableNameUpper = newValues.table_name.toUpperCase();
|
|
||||||
const newTableSchemaUpper = newValues.table_schema.toUpperCase();
|
|
||||||
const oldTableNameUpper = oldTableName.toUpperCase();
|
|
||||||
const oldTableSchemaUpper = oldTableSchema.toUpperCase();
|
|
||||||
|
|
||||||
// Nimbus table-metadata
|
|
||||||
if (nimbusId) {
|
|
||||||
await this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: newTableNameUpper, table_schema: newTableSchemaUpper });
|
|
||||||
rollbackSteps.push(() => this.catalogService.renameTableOnNimbus(nimbusUrl, nimbusId, { table_name: oldTableNameUpper, table_schema: oldTableSchemaUpper }));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Nimbus column-metadata
|
|
||||||
await this.catalogService.renameColumnMetadataOnNimbus(
|
|
||||||
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
|
|
||||||
);
|
|
||||||
rollbackSteps.push(() => this.catalogService.renameColumnMetadataOnNimbus(
|
|
||||||
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
|
|
||||||
));
|
|
||||||
|
|
||||||
// Nimbus data-preview
|
|
||||||
await this.catalogService.renameDataPreviewOnNimbus(
|
|
||||||
nimbusUrl, databaseName, oldTableNameUpper, oldTableSchemaUpper, newTableNameUpper, newTableSchemaUpper,
|
|
||||||
);
|
|
||||||
rollbackSteps.push(() => this.catalogService.renameDataPreviewOnNimbus(
|
|
||||||
nimbusUrl, databaseName, newTableNameUpper, newTableSchemaUpper, oldTableNameUpper, oldTableSchemaUpper,
|
|
||||||
));
|
|
||||||
|
|
||||||
this.logger.info(`Synced catalog rename for ${key}`, { jobId, oldTableName, newTableName: newValues.table_name });
|
|
||||||
} catch (error) {
|
|
||||||
this.logger.error(`Catalog sync failed for ${key}, rolling back catalog`, { jobId, error: error.message });
|
|
||||||
await this.executeRollback(rollbackSteps, key, jobId);
|
|
||||||
throw error;
|
|
||||||
}
|
}
|
||||||
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private buildRenameTargets(
|
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
|
||||||
body: RenameTablesBody,
|
|
||||||
outputConfig: any,
|
|
||||||
): Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> {
|
|
||||||
const DEFAULT_SCHEMAS = { raw: 'PUBLIC', qualify: 'STAGED' };
|
|
||||||
const targets: Array<{ key: string; oldTableName: string; oldTableSchema: string; newValues: { table_name: string; table_schema: string } }> = [];
|
|
||||||
|
|
||||||
for (const key of ['raw', 'qualify'] as const) {
|
|
||||||
if (!body[key]) continue;
|
|
||||||
|
|
||||||
const nested = outputConfig[key];
|
|
||||||
|
|
||||||
// qualify: only sync if output_config.qualify already exists
|
|
||||||
if (key === 'qualify' && !nested?.table_name) continue;
|
|
||||||
|
|
||||||
const oldTableName = nested?.table_name || outputConfig.table_name;
|
|
||||||
if (!oldTableName) continue;
|
|
||||||
|
|
||||||
targets.push({
|
|
||||||
key,
|
|
||||||
oldTableName,
|
|
||||||
oldTableSchema: nested?.table_schema || DEFAULT_SCHEMAS[key],
|
|
||||||
newValues: body[key],
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return targets;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async executeRollback(
|
|
||||||
steps: Array<() => Promise<void>>,
|
|
||||||
targetKey: string,
|
|
||||||
jobId: string,
|
|
||||||
): Promise<void> {
|
|
||||||
for (const rollback of steps.reverse()) {
|
|
||||||
try {
|
|
||||||
await rollback();
|
|
||||||
} catch (error) {
|
|
||||||
this.logger.error(`Rollback failed for ${targetKey}`, { jobId, error: error.message });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Get('jobs/jdbc/configs/allowed_datatypes')
|
@Get('jobs/jdbc/configs/allowed_datatypes')
|
||||||
@ApiOperation({ summary: 'Get allowed datatypes for JDBC' })
|
@ApiOperation({ summary: 'Get allowed datatypes for JDBC' })
|
||||||
@@ -1194,52 +1023,6 @@ export class PlatformApiController {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ==================== JOBS - SINGER REPLICATION ROUTES ====================
|
|
||||||
|
|
||||||
@Get('jobs/singer/:jobId')
|
|
||||||
@ApiOperation({ summary: 'Get Singer job details' })
|
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
|
||||||
async getSingerJob(@Param('jobId') jobId: string, @User() user: RequestUser) {
|
|
||||||
// Normalize job ID for Platform API (replace - with _)
|
|
||||||
const normalizedJobId = this.normalizeJobId(jobId);
|
|
||||||
return this.platformApiService.proxy('GET', `/jobs/singer/${normalizedJobId}`, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post('jobs/singer/:jobId/sync-mode')
|
|
||||||
@ApiOperation({ summary: 'Update Singer job sync mode' })
|
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
|
||||||
async updateSingerSyncMode(
|
|
||||||
@Param('jobId') jobId: string,
|
|
||||||
@Body() body: any,
|
|
||||||
@User() user: RequestUser,
|
|
||||||
) {
|
|
||||||
// Normalize job ID for Platform API (replace - with _)
|
|
||||||
const normalizedJobId = this.normalizeJobId(jobId);
|
|
||||||
|
|
||||||
const result = await this.platformApiService.proxy(
|
|
||||||
'POST',
|
|
||||||
`/jobs/singer/${normalizedJobId}/sync-mode`,
|
|
||||||
user,
|
|
||||||
body,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Sync to DynamoDB (pass raw jobId for pipeline extraction)
|
|
||||||
await this.syncSingerSyncModeToDynamoDB(jobId, body, user);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== JOBS - S3 ROUTES ====================
|
|
||||||
|
|
||||||
@Get('jobs/s3/:jobId')
|
|
||||||
@ApiOperation({ summary: 'Get S3 job details' })
|
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
|
||||||
async getS3Job(@Param('jobId') jobId: string, @User() user: RequestUser) {
|
|
||||||
// Normalize job ID for Platform API (replace - with _)
|
|
||||||
const normalizedJobId = this.normalizeJobId(jobId);
|
|
||||||
return this.platformApiService.proxy('GET', `/jobs/s3/${normalizedJobId}`, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
// ==================== HEALTH ROUTE ====================
|
// ==================== HEALTH ROUTE ====================
|
||||||
|
|
||||||
@Get('health')
|
@Get('health')
|
||||||
|
|||||||
@@ -8,9 +8,10 @@ import { ElasticsearchModule } from '../../services/elasticsearch';
|
|||||||
import { DynamoDBModule } from '../../services/dynamodb';
|
import { DynamoDBModule } from '../../services/dynamodb';
|
||||||
import { CustomersModule } from '../customers/customers.module';
|
import { CustomersModule } from '../customers/customers.module';
|
||||||
import { CatalogModule } from '../catalog/catalog.module';
|
import { CatalogModule } from '../catalog/catalog.module';
|
||||||
|
import { InputsModule } from '../inputs/inputs.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule],
|
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, InputsModule],
|
||||||
controllers: [PlatformApiController],
|
controllers: [PlatformApiController],
|
||||||
providers: [PlatformApiService, DadosferaLogger],
|
providers: [PlatformApiService, DadosferaLogger],
|
||||||
exports: [PlatformApiService],
|
exports: [PlatformApiService],
|
||||||
|
|||||||
@@ -125,9 +125,13 @@ export class DynamoDBService {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const { Item } = await this.documentClient.send(getCommand);
|
try {
|
||||||
|
const { Item } = await this.documentClient.send(getCommand);
|
||||||
return Item as InputDocument | null;
|
return Item as InputDocument | null;
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('DynamoDB: findInput failed', { inputId, clientId, error: error.message });
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async deleteInput(clientId: string, inputId: string): Promise<void> {
|
async deleteInput(clientId: string, inputId: string): Promise<void> {
|
||||||
@@ -203,7 +207,6 @@ export class DynamoDBService {
|
|||||||
updatedTable.reference_column = changes.reference_column;
|
updatedTable.reference_column = changes.reference_column;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
tables[tableIndex] = updatedTable;
|
tables[tableIndex] = updatedTable;
|
||||||
|
|
||||||
// Save updated document
|
// Save updated document
|
||||||
@@ -231,4 +234,5 @@ export class DynamoDBService {
|
|||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user