mirror of
https://github.com/dadosfera/maestro.git
synced 2026-08-31 19:58:21 +00:00
UPDATE: one platform route for table removal; table orchestration moves to PipelineTablesService
Review (maestro #510): batch and CDC tables are both removed through DELETE /pipeline/{id}/jobs (the platform dispatches by type: Airflow refresh vs Kafka Connect reconfigure), so the connector 'if' in the controller is gone. deleteTable/deleteTables/addTable are now thin controller methods over PipelineTablesService (mark -> resolve jobs -> platform -> rollback), with typed request bodies. Behavior change for batch: the platform refuses to remove the LAST table of a pipeline (400 'Cannot remove all jobs'), where DELETE /jobs/{id} allowed it. Co-Authored-By: WOZCODE <contact@withwoz.com>
This commit is contained in:
@@ -0,0 +1,243 @@
|
||||
// These service modules pull in gRPC client-config modules that read
|
||||
// process.env at load time; mock them (hoisted before imports) so the spec
|
||||
// needs no runtime env.
|
||||
jest.mock('../inputs/inputs.service', () => ({ InputsService: class {} }));
|
||||
jest.mock('../pipelinesV2/pipelines.service', () => ({ PipelinesService: class {} }));
|
||||
|
||||
import { NotFoundException } from '@nestjs/common';
|
||||
import { PipelineTablesService } from './pipeline-tables.service';
|
||||
|
||||
const logger = { info: jest.fn(), error: jest.fn() };
|
||||
const user = { customer_id: 'c1', customer_name: 'cust', user_id: 'u1' } as never;
|
||||
const info = { customer_id: 'c1', customer: 'cust', user_id: 'u1' };
|
||||
|
||||
type Mocks = {
|
||||
proxy: jest.Mock;
|
||||
markTableDeleted: jest.Mock;
|
||||
unmarkTableDeleted: jest.Mock;
|
||||
addCdcTable: jest.Mock;
|
||||
removeCdcTable: jest.Mock;
|
||||
addCdcJobs: jest.Mock;
|
||||
};
|
||||
|
||||
function build(): { service: PipelineTablesService; m: Mocks } {
|
||||
const m: Mocks = {
|
||||
proxy: jest.fn(),
|
||||
markTableDeleted: jest.fn().mockResolvedValue({ is_deleted: true, deleted_at: 't' }),
|
||||
unmarkTableDeleted: jest.fn().mockResolvedValue({}),
|
||||
addCdcTable: jest.fn().mockResolvedValue({ input: {} }),
|
||||
removeCdcTable: jest.fn().mockResolvedValue({ input: {} }),
|
||||
addCdcJobs: jest.fn(),
|
||||
};
|
||||
type Deps = ConstructorParameters<typeof PipelineTablesService>;
|
||||
const service = new PipelineTablesService(
|
||||
{ proxy: m.proxy } as unknown as Deps[0],
|
||||
{
|
||||
markTableDeleted: m.markTableDeleted,
|
||||
unmarkTableDeleted: m.unmarkTableDeleted,
|
||||
addCdcTable: m.addCdcTable,
|
||||
removeCdcTable: m.removeCdcTable,
|
||||
} as unknown as Deps[1],
|
||||
{ addCdcJobs: m.addCdcJobs } as unknown as Deps[2],
|
||||
{ logger } as unknown as Deps[3],
|
||||
);
|
||||
return { service, m };
|
||||
}
|
||||
|
||||
const pipelineWithJobs = (connector: string) => ({
|
||||
jobs: [
|
||||
{ job_id: 'p_0', input: { connector, table_name: 'pedidos' } },
|
||||
{ job_id: 'p_1', input: { connector, table_name: 'clientes' } },
|
||||
{ job_id: 'p_2', input: { connector, table_name: 'produtos' } },
|
||||
],
|
||||
});
|
||||
|
||||
const deleteCalls = (proxy: jest.Mock) =>
|
||||
proxy.mock.calls.filter(([method]) => method === 'DELETE');
|
||||
|
||||
describe('PipelineTablesService.removeTable', () => {
|
||||
afterEach(() => jest.clearAllMocks());
|
||||
|
||||
it.each(['cdc', 'jdbc'])(
|
||||
'removes a %s table through the single pipeline jobs route',
|
||||
async (connector) => {
|
||||
const { service, m } = build();
|
||||
m.proxy.mockImplementation((method: string) =>
|
||||
Promise.resolve(method === 'GET' ? pipelineWithJobs(connector) : {}),
|
||||
);
|
||||
|
||||
const result = await service.removeTable('pi-d', 'iid', 'pedidos', user);
|
||||
|
||||
expect(m.markTableDeleted).toHaveBeenCalledWith({ input_id: 'iid', table_name: 'pedidos', info });
|
||||
expect(m.proxy).toHaveBeenCalledWith('GET', '/pipeline/pi_d', user);
|
||||
expect(deleteCalls(m.proxy)).toEqual([
|
||||
['DELETE', '/pipeline/pi_d/jobs', user, { job_ids: ['p_0'], delete_snowflake_tables: false }],
|
||||
]);
|
||||
expect(result).toEqual({ name: 'pedidos', is_deleted: true, deleted_at: 't' });
|
||||
expect(m.unmarkTableDeleted).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('rolls back the mark when the platform delete fails', async () => {
|
||||
const { service, m } = build();
|
||||
m.proxy.mockImplementation((method: string) =>
|
||||
method === 'GET' ? Promise.resolve(pipelineWithJobs('cdc')) : Promise.reject(new Error('platform boom')),
|
||||
);
|
||||
|
||||
await expect(service.removeTable('pid', 'iid', 'pedidos', user)).rejects.toThrow('platform boom');
|
||||
expect(m.unmarkTableDeleted).toHaveBeenCalledWith({ input_id: 'iid', table_name: 'pedidos', info });
|
||||
});
|
||||
|
||||
it('404s (and rolls back) when the table has no job', async () => {
|
||||
const { service, m } = build();
|
||||
m.proxy.mockResolvedValue(pipelineWithJobs('cdc'));
|
||||
|
||||
await expect(service.removeTable('pid', 'iid', 'ghost', user)).rejects.toBeInstanceOf(NotFoundException);
|
||||
expect(deleteCalls(m.proxy)).toHaveLength(0);
|
||||
expect(m.unmarkTableDeleted).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PipelineTablesService.removeTables', () => {
|
||||
afterEach(() => jest.clearAllMocks());
|
||||
|
||||
it('removes N tables in ONE platform call (connectors reconfigured once)', async () => {
|
||||
const { service, m } = build();
|
||||
m.proxy.mockImplementation((method: string) =>
|
||||
Promise.resolve(method === 'GET' ? pipelineWithJobs('cdc') : {}),
|
||||
);
|
||||
|
||||
const result = await service.removeTables('pid', 'iid', ['pedidos', 'produtos'], user);
|
||||
|
||||
expect(m.markTableDeleted).toHaveBeenCalledTimes(2);
|
||||
expect(deleteCalls(m.proxy)).toEqual([
|
||||
['DELETE', '/pipeline/pid/jobs', user, { job_ids: ['p_0', 'p_2'], delete_snowflake_tables: false }],
|
||||
]);
|
||||
expect(result).toEqual({ table_names: ['pedidos', 'produtos'], deleted: true });
|
||||
});
|
||||
|
||||
it("rolls back only this call's marks when the platform delete fails", async () => {
|
||||
const { service, m } = build();
|
||||
m.proxy.mockImplementation((method: string) =>
|
||||
method === 'GET' ? Promise.resolve(pipelineWithJobs('cdc')) : Promise.reject(new Error('platform boom')),
|
||||
);
|
||||
|
||||
await expect(service.removeTables('pid', 'iid', ['pedidos', 'clientes'], user)).rejects.toThrow();
|
||||
|
||||
const unmarked = m.unmarkTableDeleted.mock.calls.map(([arg]) => arg.table_name).sort();
|
||||
expect(unmarked).toEqual(['clientes', 'pedidos']);
|
||||
});
|
||||
|
||||
it('rolls back the marks made so far if a later mark fails (atomic)', async () => {
|
||||
const { service, m } = build();
|
||||
m.markTableDeleted
|
||||
.mockResolvedValueOnce({ is_deleted: true, deleted_at: 't' })
|
||||
.mockRejectedValueOnce(new Error('dynamo boom'));
|
||||
|
||||
await expect(service.removeTables('pid', 'iid', ['pedidos', 'clientes'], user)).rejects.toThrow('dynamo boom');
|
||||
|
||||
expect(m.unmarkTableDeleted).toHaveBeenCalledTimes(1);
|
||||
expect(m.unmarkTableDeleted.mock.calls[0][0].table_name).toBe('pedidos');
|
||||
expect(m.proxy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('404s when a requested table has no matching job and rolls back the marks', async () => {
|
||||
const { service, m } = build();
|
||||
m.proxy.mockResolvedValue(pipelineWithJobs('cdc'));
|
||||
|
||||
await expect(service.removeTables('pid', 'iid', ['pedidos', 'ghost'], user)).rejects.toBeInstanceOf(NotFoundException);
|
||||
expect(m.unmarkTableDeleted).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('keeps going when a rollback step itself fails', async () => {
|
||||
const { service, m } = build();
|
||||
m.proxy.mockImplementation((method: string) =>
|
||||
method === 'GET' ? Promise.resolve(pipelineWithJobs('cdc')) : Promise.reject(new Error('platform boom')),
|
||||
);
|
||||
m.unmarkTableDeleted.mockRejectedValueOnce(new Error('unmark boom'));
|
||||
|
||||
await expect(service.removeTables('pid', 'iid', ['pedidos', 'clientes'], user)).rejects.toThrow('platform boom');
|
||||
expect(m.unmarkTableDeleted).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PipelineTablesService.addTable', () => {
|
||||
afterEach(() => jest.clearAllMocks());
|
||||
|
||||
const body = {
|
||||
table_name: 'orders',
|
||||
table_schema: 'public',
|
||||
primary_keys: ['id'],
|
||||
destinations: {
|
||||
raw: { table_schema: 'raw', table_name: 'orders' },
|
||||
qualify: { table_schema: 'qualify', table_name: 'orders' },
|
||||
},
|
||||
};
|
||||
|
||||
const storedTable = (extra: Record<string, unknown> = {}) => ({
|
||||
table_schema: 'public',
|
||||
table_name: 'orders',
|
||||
name: 'orders',
|
||||
primary_keys: ['id'],
|
||||
iceberg_table_name: undefined,
|
||||
iceberg_qualify_table_name: undefined,
|
||||
columns: [],
|
||||
column_exclude_list: [],
|
||||
...extra,
|
||||
});
|
||||
|
||||
it('appends to DynamoDB, then dispatches the platform jobs', async () => {
|
||||
const { service, m } = build();
|
||||
m.addCdcJobs.mockResolvedValue({ job_ids: ['p_2'], skipped: [] });
|
||||
|
||||
const result = await service.addTable('pid', 'iid', body, user);
|
||||
|
||||
expect(m.addCdcTable).toHaveBeenCalledWith({ client_id: 'c1', id: 'iid', table: storedTable(), info });
|
||||
expect(m.addCdcJobs).toHaveBeenCalledWith({
|
||||
pipeline_id: 'pid',
|
||||
input_id: 'iid',
|
||||
tables: [{ table_schema: 'public', table_name: 'orders', primary_keys: ['id'], destinations: body.destinations }],
|
||||
info,
|
||||
});
|
||||
expect(result).toEqual({ job_ids: ['p_2'], skipped: [] });
|
||||
expect(m.removeCdcTable).not.toHaveBeenCalled();
|
||||
expect(m.addCdcTable.mock.invocationCallOrder[0]).toBeLessThan(m.addCdcJobs.mock.invocationCallOrder[0]);
|
||||
});
|
||||
|
||||
it('carries the iceberg names and columns through to the stored CdcTable', async () => {
|
||||
const { service, m } = build();
|
||||
m.addCdcJobs.mockResolvedValue({ job_ids: ['p_3'], skipped: [] });
|
||||
const columns = [
|
||||
{ name: 'id', type: 'int', is_primary_key: true },
|
||||
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
|
||||
];
|
||||
|
||||
await service.addTable('pid', 'iid', {
|
||||
...body,
|
||||
iceberg_table_name: 'cdc_raw.public__orders',
|
||||
iceberg_qualify_table_name: 'orders_dedup',
|
||||
columns,
|
||||
column_exclude_list: ['descr'],
|
||||
}, user);
|
||||
|
||||
expect(m.addCdcTable).toHaveBeenCalledWith({
|
||||
client_id: 'c1',
|
||||
id: 'iid',
|
||||
table: storedTable({
|
||||
iceberg_table_name: 'cdc_raw.public__orders',
|
||||
iceberg_qualify_table_name: 'orders_dedup',
|
||||
columns,
|
||||
column_exclude_list: ['descr'],
|
||||
}),
|
||||
info,
|
||||
});
|
||||
});
|
||||
|
||||
it('rolls back the DynamoDB row when AddJobs fails', async () => {
|
||||
const { service, m } = build();
|
||||
m.addCdcJobs.mockRejectedValue(new Error('grpc boom'));
|
||||
|
||||
await expect(service.addTable('pid', 'iid', body, user)).rejects.toThrow('grpc boom');
|
||||
expect(m.removeCdcTable).toHaveBeenCalledWith({ client_id: 'c1', id: 'iid', table_name: 'orders', info });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,157 @@
|
||||
import { Inject, Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||
|
||||
import { RequestUser } from '../../decorators/user.decorator';
|
||||
import { InputsService } from '../inputs/inputs.service';
|
||||
import { toCdcTable } from '../inputs/cdc-table.mapper';
|
||||
import { PipelinesService } from '../pipelinesV2/pipelines.service';
|
||||
import { PlatformApiService } from './platform-api.service';
|
||||
import { AddCdcTableBody } from './platform-api.dto';
|
||||
|
||||
interface PlatformJob {
|
||||
job_id: string;
|
||||
input?: { table_name?: string; connector?: string } | null;
|
||||
}
|
||||
|
||||
interface PlatformPipeline {
|
||||
jobs?: PlatformJob[] | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add/remove tables of a pipeline: the DynamoDB input (soft-delete flags or
|
||||
* a new CdcTable row) and the platform-api jobs move together, with the
|
||||
* DynamoDB side rolled back when the platform side fails.
|
||||
*
|
||||
* Removal always goes through DELETE /pipeline/{id}/jobs — the one platform
|
||||
* route that dispatches by pipeline type (Airflow refresh for batch, Kafka
|
||||
* Connect reconfiguration for CDC). Landed destination data is kept
|
||||
* (delete_snowflake_tables: false), matching the soft-delete in DynamoDB.
|
||||
*/
|
||||
@Injectable()
|
||||
export class PipelineTablesService {
|
||||
private logger: DadosferaLogger['logger'];
|
||||
|
||||
constructor(
|
||||
private readonly platformApiService: PlatformApiService,
|
||||
private readonly inputsService: InputsService,
|
||||
private readonly pipelinesClientService: PipelinesService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
}
|
||||
|
||||
async removeTable(pipelineId: string, inputId: string, tableName: string, user: RequestUser) {
|
||||
const info = this.infoOf(user);
|
||||
|
||||
this.logger.info('removeTable: marking table as deleted', { inputId, tableName });
|
||||
const marked = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info });
|
||||
|
||||
try {
|
||||
const jobIds = await this.resolveJobIds(pipelineId, [tableName], user);
|
||||
await this.removeJobs(pipelineId, jobIds, user);
|
||||
return { name: tableName, is_deleted: marked.is_deleted ?? true, deleted_at: marked.deleted_at };
|
||||
} catch (error) {
|
||||
this.logger.error('removeTable: platform-api delete failed, rolling back the mark', { tableName, error: error.message });
|
||||
await this.unmarkAll(inputId, [tableName], info);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Removes N tables with ONE platform call, so CDC connectors are reconfigured once. */
|
||||
async removeTables(pipelineId: string, inputId: string, tableNames: string[], user: RequestUser) {
|
||||
const info = this.infoOf(user);
|
||||
|
||||
// Soft-delete each table, tracking which succeeded so a later failure
|
||||
// only rolls back the marks made in THIS call.
|
||||
const marked: string[] = [];
|
||||
try {
|
||||
for (const name of tableNames) {
|
||||
await this.inputsService.markTableDeleted({ input_id: inputId, table_name: name, info });
|
||||
marked.push(name);
|
||||
}
|
||||
|
||||
const jobIds = await this.resolveJobIds(pipelineId, tableNames, user);
|
||||
await this.removeJobs(pipelineId, jobIds, user);
|
||||
return { table_names: tableNames, deleted: true };
|
||||
} catch (error) {
|
||||
this.logger.error("removeTables: failed, rolling back this call's marks", { tableNames, error: error.message });
|
||||
await this.unmarkAll(inputId, marked, info);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/** Appends a CDC table to the DynamoDB input, then dispatches its platform jobs. */
|
||||
async addTable(pipelineId: string, inputId: string, body: AddCdcTableBody, user: RequestUser) {
|
||||
const info = this.infoOf(user);
|
||||
const cdcTable = toCdcTable(body);
|
||||
|
||||
this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });
|
||||
await this.inputsService.addCdcTable({ client_id: user.customer_id, id: inputId, table: cdcTable, info });
|
||||
|
||||
try {
|
||||
return await this.pipelinesClientService.addCdcJobs({
|
||||
pipeline_id: pipelineId,
|
||||
input_id: inputId,
|
||||
tables: [{
|
||||
table_schema: body.table_schema,
|
||||
table_name: body.table_name,
|
||||
primary_keys: body.primary_keys,
|
||||
destinations: body.destinations,
|
||||
}],
|
||||
info,
|
||||
});
|
||||
} catch (error) {
|
||||
this.logger.error('addTable: platform AddJobs failed, rolling back the DynamoDB row', { tableName: body.table_name, error: error.message });
|
||||
try {
|
||||
await this.inputsService.removeCdcTable({ client_id: user.customer_id, id: inputId, table_name: body.table_name, info });
|
||||
} catch (rollbackError) {
|
||||
this.logger.error('addTable: rollback failed', { error: rollbackError.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private infoOf(user: RequestUser): Info {
|
||||
return { customer_id: user.customer_id, customer: user.customer_name, user_id: user.user_id };
|
||||
}
|
||||
|
||||
/** Platform-API replaces '-' with '_' in pipeline IDs. */
|
||||
private normalizePipelineId(id: string): string {
|
||||
return id.replace(/-/g, '_');
|
||||
}
|
||||
|
||||
private async resolveJobIds(pipelineId: string, tableNames: string[], user: RequestUser): Promise<string[]> {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const pipeline: PlatformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
|
||||
const jobs = pipeline?.jobs ?? [];
|
||||
|
||||
return tableNames.map((name) => {
|
||||
const job = jobs.find((j) => j.input?.table_name === name);
|
||||
if (!job) throw new NotFoundException(`Job for table '${name}' not found in pipeline`);
|
||||
return job.job_id;
|
||||
});
|
||||
}
|
||||
|
||||
private async removeJobs(pipelineId: string, jobIds: string[], user: RequestUser) {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
this.logger.info('removeJobs: deleting jobs from platform-api', { jobIds });
|
||||
await this.platformApiService.proxy(
|
||||
'DELETE',
|
||||
`/pipeline/${normalizedPipelineId}/jobs`,
|
||||
user,
|
||||
{ job_ids: jobIds, delete_snowflake_tables: false },
|
||||
);
|
||||
this.logger.info('removeJobs: jobs deleted', { jobIds });
|
||||
}
|
||||
|
||||
private async unmarkAll(inputId: string, tableNames: string[], info: Info) {
|
||||
for (const name of tableNames) {
|
||||
try {
|
||||
await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: name, info });
|
||||
} catch (rollbackError) {
|
||||
this.logger.error('rollback failed: table stays marked as deleted', { tableName: name, error: rollbackError.message });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,53 +5,43 @@
|
||||
jest.mock('../customers/customers.service', () => ({ CustomersService: class {} }));
|
||||
jest.mock('../catalog/catalog.service', () => ({ CatalogService: class {} }));
|
||||
jest.mock('../inputs/inputs.service', () => ({ InputsService: class {} }));
|
||||
jest.mock('../pipelinesV2/pipelines.service', () => ({ PipelinesService: class {} }));
|
||||
jest.mock('./pipeline-tables.service', () => ({ PipelineTablesService: class {} }));
|
||||
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { Test, TestingModule } from '@nestjs/testing';
|
||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { PlatformApiController } from './platform-api.controller';
|
||||
import { PlatformApiService } from './platform-api.service';
|
||||
import { PipelineTablesService } from './pipeline-tables.service';
|
||||
import { ElasticsearchService } from '../../services/elasticsearch';
|
||||
import { DynamoDBService } from '../../services/dynamodb';
|
||||
import { CustomersService } from '../customers/customers.service';
|
||||
import { CatalogService } from '../catalog/catalog.service';
|
||||
import { InputsService } from '../inputs/inputs.service';
|
||||
import { PipelinesService } from '../pipelinesV2/pipelines.service';
|
||||
|
||||
const logger = {
|
||||
info: (...args) => args,
|
||||
error: (...args) => args,
|
||||
};
|
||||
const logger = { info: jest.fn(), error: jest.fn() };
|
||||
const mockUser = { customer_id: 'c1', customer_name: 'cust', user_id: 'u1' } as never;
|
||||
|
||||
const mockUser: any = {
|
||||
customer_id: 'c1',
|
||||
customer_name: 'cust',
|
||||
user_id: 'u1',
|
||||
};
|
||||
|
||||
describe('PlatformApiController - deleteTable', () => {
|
||||
// The table routes are thin: validate the body, delegate to
|
||||
// PipelineTablesService (covered in pipeline-tables.service.spec.ts).
|
||||
describe('PlatformApiController - table routes', () => {
|
||||
let controller: PlatformApiController;
|
||||
let platformApiService: { proxy: jest.Mock };
|
||||
let inputsService: { markTableDeleted: jest.Mock; unmarkTableDeleted: jest.Mock };
|
||||
let tables: { removeTable: jest.Mock; removeTables: jest.Mock; addTable: jest.Mock };
|
||||
|
||||
beforeEach(async () => {
|
||||
platformApiService = { proxy: jest.fn() };
|
||||
inputsService = {
|
||||
markTableDeleted: jest.fn(),
|
||||
unmarkTableDeleted: jest.fn(),
|
||||
};
|
||||
tables = { removeTable: jest.fn(), removeTables: jest.fn(), addTable: jest.fn() };
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
controllers: [PlatformApiController],
|
||||
providers: [
|
||||
{ provide: PlatformApiService, useValue: platformApiService },
|
||||
{ provide: PlatformApiService, useValue: {} },
|
||||
{ provide: ElasticsearchService, useValue: {} },
|
||||
{ provide: DynamoDBService, useValue: {} },
|
||||
{ provide: CustomersService, useValue: {} },
|
||||
{ provide: CatalogService, useValue: {} },
|
||||
{ provide: InputsService, useValue: inputsService },
|
||||
{ provide: PipelinesService, useValue: {} },
|
||||
{ provide: InputsService, useValue: {} },
|
||||
{ provide: PipelineTablesService, useValue: tables },
|
||||
{ provide: DadosferaLogger, useValue: { logger } },
|
||||
],
|
||||
}).compile();
|
||||
@@ -59,318 +49,37 @@ describe('PlatformApiController - deleteTable', () => {
|
||||
controller = module.get<PlatformApiController>(PlatformApiController);
|
||||
});
|
||||
|
||||
it('CDC removal reconfigures the connector', async () => {
|
||||
inputsService.markTableDeleted.mockResolvedValue({ is_deleted: true, deleted_at: 't' });
|
||||
platformApiService.proxy.mockImplementation((method: string, path: string) => {
|
||||
if (method === 'GET') {
|
||||
return Promise.resolve({
|
||||
jobs: [{ job_id: 'p_0', input: { connector: 'cdc', table_name: 'pedidos' } }],
|
||||
});
|
||||
}
|
||||
return Promise.resolve({});
|
||||
});
|
||||
|
||||
await controller.deleteTable('pid', 'iid', { table_name: 'pedidos' }, mockUser);
|
||||
|
||||
expect(platformApiService.proxy).toHaveBeenCalledWith(
|
||||
'DELETE',
|
||||
'/pipeline/pid/jobs',
|
||||
mockUser,
|
||||
{ job_ids: ['p_0'], delete_snowflake_tables: false },
|
||||
);
|
||||
const deletedViaJobsRoute = platformApiService.proxy.mock.calls.some(
|
||||
([method, path]: any[]) => method === 'DELETE' && path === '/jobs/p_0',
|
||||
);
|
||||
expect(deletedViaJobsRoute).toBe(false);
|
||||
it('deleteTable delegates', async () => {
|
||||
tables.removeTable.mockResolvedValue({ name: 'pedidos', is_deleted: true });
|
||||
await expect(controller.deleteTable('pid', 'iid', { table_name: 'pedidos' }, mockUser))
|
||||
.resolves.toEqual({ name: 'pedidos', is_deleted: true });
|
||||
expect(tables.removeTable).toHaveBeenCalledWith('pid', 'iid', 'pedidos', mockUser);
|
||||
});
|
||||
|
||||
it('batch removal unchanged', async () => {
|
||||
inputsService.markTableDeleted.mockResolvedValue({ is_deleted: true, deleted_at: 't' });
|
||||
platformApiService.proxy.mockImplementation((method: string, path: string) => {
|
||||
if (method === 'GET') {
|
||||
return Promise.resolve({
|
||||
jobs: [{ job_id: 'p_0', input: { connector: 'jdbc', table_name: 'pedidos' } }],
|
||||
});
|
||||
}
|
||||
return Promise.resolve({});
|
||||
});
|
||||
|
||||
await controller.deleteTable('pid', 'iid', { table_name: 'pedidos' }, mockUser);
|
||||
|
||||
expect(platformApiService.proxy).toHaveBeenCalledWith(
|
||||
'DELETE',
|
||||
'/jobs/p_0',
|
||||
mockUser,
|
||||
);
|
||||
const reconfiguredConnector = platformApiService.proxy.mock.calls.some(
|
||||
([method, path]: any[]) => method === 'DELETE' && path === '/pipeline/pid/jobs',
|
||||
);
|
||||
expect(reconfiguredConnector).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PlatformApiController - addTable', () => {
|
||||
let controller: PlatformApiController;
|
||||
let inputsService: { addCdcTable: jest.Mock; removeCdcTable: jest.Mock };
|
||||
let pipelinesClientService: { addCdcJobs: jest.Mock };
|
||||
|
||||
const body = {
|
||||
table_name: 'orders',
|
||||
table_schema: 'public',
|
||||
primary_keys: ['id'],
|
||||
destinations: {
|
||||
raw: { table_schema: 'raw', table_name: 'orders' },
|
||||
qualify: { table_schema: 'qualify', table_name: 'orders' },
|
||||
},
|
||||
};
|
||||
|
||||
beforeEach(async () => {
|
||||
inputsService = {
|
||||
addCdcTable: jest.fn(),
|
||||
removeCdcTable: jest.fn(),
|
||||
};
|
||||
pipelinesClientService = {
|
||||
addCdcJobs: jest.fn(),
|
||||
};
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
controllers: [PlatformApiController],
|
||||
providers: [
|
||||
{ provide: PlatformApiService, useValue: { proxy: jest.fn() } },
|
||||
{ provide: ElasticsearchService, useValue: {} },
|
||||
{ provide: DynamoDBService, useValue: {} },
|
||||
{ provide: CustomersService, useValue: {} },
|
||||
{ provide: CatalogService, useValue: {} },
|
||||
{ provide: InputsService, useValue: inputsService },
|
||||
{ provide: PipelinesService, useValue: pipelinesClientService },
|
||||
{ provide: DadosferaLogger, useValue: { logger } },
|
||||
],
|
||||
}).compile();
|
||||
|
||||
controller = module.get<PlatformApiController>(PlatformApiController);
|
||||
it('deleteTables delegates', async () => {
|
||||
tables.removeTables.mockResolvedValue({ deleted: true });
|
||||
await controller.deleteTables('pid', 'iid', { table_names: ['a', 'b'] }, mockUser);
|
||||
expect(tables.removeTables).toHaveBeenCalledWith('pid', 'iid', ['a', 'b'], mockUser);
|
||||
});
|
||||
|
||||
it('addTable: DynamoDB append then platform AddJobs, returns job_ids', async () => {
|
||||
inputsService.addCdcTable.mockResolvedValue({ input: {} });
|
||||
pipelinesClientService.addCdcJobs.mockResolvedValue({ job_ids: ['p_2'], skipped: [] });
|
||||
|
||||
const result = await controller.addTable('pid', 'iid', body, mockUser);
|
||||
|
||||
expect(inputsService.addCdcTable).toHaveBeenCalledWith({
|
||||
client_id: 'c1',
|
||||
id: 'iid',
|
||||
table: {
|
||||
table_schema: 'public',
|
||||
table_name: 'orders',
|
||||
primary_keys: ['id'],
|
||||
name: 'orders',
|
||||
iceberg_table_name: undefined,
|
||||
iceberg_qualify_table_name: undefined,
|
||||
columns: [],
|
||||
column_exclude_list: [],
|
||||
},
|
||||
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
|
||||
});
|
||||
expect(pipelinesClientService.addCdcJobs).toHaveBeenCalledWith({
|
||||
pipeline_id: 'pid',
|
||||
input_id: 'iid',
|
||||
tables: [{
|
||||
table_schema: 'public',
|
||||
table_name: 'orders',
|
||||
primary_keys: ['id'],
|
||||
destinations: body.destinations,
|
||||
}],
|
||||
info: { customer_id: 'c1', user_id: 'u1', customer: 'cust' },
|
||||
});
|
||||
expect(result).toEqual({ job_ids: ['p_2'], skipped: [] });
|
||||
expect(inputsService.removeCdcTable).not.toHaveBeenCalled();
|
||||
|
||||
const addCdcTableOrder = inputsService.addCdcTable.mock.invocationCallOrder[0];
|
||||
const addCdcJobsOrder = pipelinesClientService.addCdcJobs.mock.invocationCallOrder[0];
|
||||
expect(addCdcTableOrder).toBeLessThan(addCdcJobsOrder);
|
||||
it('deleteTables rejects an empty table_names before touching anything', async () => {
|
||||
await expect(controller.deleteTables('pid', 'iid', { table_names: [] }, mockUser))
|
||||
.rejects.toBeInstanceOf(BadRequestException);
|
||||
expect(tables.removeTables).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('addTable: carries iceberg_table_name on the added table through to AddCdcTable', async () => {
|
||||
inputsService.addCdcTable.mockResolvedValue({ input: {} });
|
||||
pipelinesClientService.addCdcJobs.mockResolvedValue({ job_ids: ['p_3'], skipped: [] });
|
||||
|
||||
const icebergBody = { ...body, iceberg_table_name: 'cdc_raw.public__orders' };
|
||||
|
||||
await controller.addTable('pid', 'iid', icebergBody, mockUser);
|
||||
|
||||
expect(inputsService.addCdcTable).toHaveBeenCalledWith({
|
||||
client_id: 'c1',
|
||||
id: 'iid',
|
||||
table: {
|
||||
table_schema: 'public',
|
||||
table_name: 'orders',
|
||||
primary_keys: ['id'],
|
||||
name: 'orders',
|
||||
iceberg_table_name: 'cdc_raw.public__orders',
|
||||
iceberg_qualify_table_name: undefined,
|
||||
columns: [],
|
||||
column_exclude_list: [],
|
||||
},
|
||||
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
|
||||
});
|
||||
});
|
||||
|
||||
it('addTable: carries columns on the added table through to AddCdcTable', async () => {
|
||||
inputsService.addCdcTable.mockResolvedValue({ input: {} });
|
||||
pipelinesClientService.addCdcJobs.mockResolvedValue({ job_ids: ['p_4'], skipped: [] });
|
||||
|
||||
const columnsBody = {
|
||||
...body,
|
||||
columns: [
|
||||
{ name: 'id', type: 'int', is_primary_key: true },
|
||||
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
|
||||
],
|
||||
};
|
||||
|
||||
await controller.addTable('pid', 'iid', columnsBody, mockUser);
|
||||
|
||||
expect(inputsService.addCdcTable).toHaveBeenCalledWith({
|
||||
client_id: 'c1',
|
||||
id: 'iid',
|
||||
table: {
|
||||
table_schema: 'public',
|
||||
table_name: 'orders',
|
||||
primary_keys: ['id'],
|
||||
name: 'orders',
|
||||
iceberg_table_name: undefined,
|
||||
iceberg_qualify_table_name: undefined,
|
||||
columns: [
|
||||
{ name: 'id', type: 'int', is_primary_key: true },
|
||||
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
|
||||
],
|
||||
column_exclude_list: [],
|
||||
},
|
||||
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
|
||||
});
|
||||
});
|
||||
|
||||
it('addTable: rolls back the DynamoDB row when AddJobs fails', async () => {
|
||||
inputsService.addCdcTable.mockResolvedValue({ input: {} });
|
||||
pipelinesClientService.addCdcJobs.mockRejectedValue(new Error('platform down'));
|
||||
inputsService.removeCdcTable.mockResolvedValue({});
|
||||
|
||||
await expect(controller.addTable('pid', 'iid', body, mockUser)).rejects.toThrow('platform down');
|
||||
|
||||
expect(inputsService.removeCdcTable).toHaveBeenCalledWith({
|
||||
client_id: 'c1',
|
||||
id: 'iid',
|
||||
it('addTable delegates', async () => {
|
||||
const body = {
|
||||
table_name: 'orders',
|
||||
info: { customer_id: 'c1', customer: 'cust', user_id: 'u1' },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
describe('PlatformApiController - deleteTables (batch)', () => {
|
||||
let controller: PlatformApiController;
|
||||
let platformApiService: { proxy: jest.Mock };
|
||||
let inputsService: { markTableDeleted: jest.Mock; unmarkTableDeleted: jest.Mock };
|
||||
|
||||
beforeEach(async () => {
|
||||
platformApiService = { proxy: jest.fn() };
|
||||
inputsService = {
|
||||
markTableDeleted: jest.fn().mockResolvedValue({ is_deleted: true, deleted_at: 't' }),
|
||||
unmarkTableDeleted: jest.fn().mockResolvedValue({}),
|
||||
table_schema: 'public',
|
||||
primary_keys: ['id'],
|
||||
destinations: {
|
||||
raw: { table_schema: 'raw', table_name: 'orders' },
|
||||
qualify: { table_schema: 'qualify', table_name: 'orders' },
|
||||
},
|
||||
};
|
||||
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
controllers: [PlatformApiController],
|
||||
providers: [
|
||||
{ provide: PlatformApiService, useValue: platformApiService },
|
||||
{ provide: ElasticsearchService, useValue: {} },
|
||||
{ provide: DynamoDBService, useValue: {} },
|
||||
{ provide: CustomersService, useValue: {} },
|
||||
{ provide: CatalogService, useValue: {} },
|
||||
{ provide: InputsService, useValue: inputsService },
|
||||
{ provide: PipelinesService, useValue: {} },
|
||||
{ provide: DadosferaLogger, useValue: { logger } },
|
||||
],
|
||||
}).compile();
|
||||
|
||||
controller = module.get<PlatformApiController>(PlatformApiController);
|
||||
});
|
||||
|
||||
const pipelineWithJobs = () => ({
|
||||
jobs: [
|
||||
{ job_id: 'p_0', input: { connector: 'cdc', table_name: 'pedidos' } },
|
||||
{ job_id: 'p_1', input: { connector: 'cdc', table_name: 'clientes' } },
|
||||
{ job_id: 'p_2', input: { connector: 'cdc', table_name: 'produtos' } },
|
||||
],
|
||||
});
|
||||
|
||||
it('removes N tables in ONE platform call (connectors reconfigured once)', async () => {
|
||||
platformApiService.proxy.mockImplementation((method: string) =>
|
||||
Promise.resolve(method === 'GET' ? pipelineWithJobs() : {}),
|
||||
);
|
||||
|
||||
await controller.deleteTables('pid', 'iid', { table_names: ['pedidos', 'produtos'] }, mockUser);
|
||||
|
||||
// one mark per table
|
||||
expect(inputsService.markTableDeleted).toHaveBeenCalledTimes(2);
|
||||
|
||||
// exactly one DELETE to the batch endpoint, with BOTH job_ids
|
||||
const deleteCalls = platformApiService.proxy.mock.calls.filter(
|
||||
([m, p]: any[]) => m === 'DELETE' && p === '/pipeline/pid/jobs',
|
||||
);
|
||||
expect(deleteCalls).toHaveLength(1);
|
||||
expect(deleteCalls[0][3]).toEqual({
|
||||
job_ids: ['p_0', 'p_2'],
|
||||
delete_snowflake_tables: false,
|
||||
});
|
||||
// never the per-job route
|
||||
const perJob = platformApiService.proxy.mock.calls.some(
|
||||
([m, p]: any[]) => m === 'DELETE' && String(p).startsWith('/jobs/'),
|
||||
);
|
||||
expect(perJob).toBe(false);
|
||||
});
|
||||
|
||||
it('rolls back only this call\'s marks when the platform delete fails', async () => {
|
||||
platformApiService.proxy.mockImplementation((method: string) => {
|
||||
if (method === 'GET') return Promise.resolve(pipelineWithJobs());
|
||||
return Promise.reject(new Error('platform boom'));
|
||||
});
|
||||
|
||||
await expect(
|
||||
controller.deleteTables('pid', 'iid', { table_names: ['pedidos', 'clientes'] }, mockUser),
|
||||
).rejects.toThrow();
|
||||
|
||||
// both marks rolled back, nothing else
|
||||
expect(inputsService.unmarkTableDeleted).toHaveBeenCalledTimes(2);
|
||||
const unmarked = inputsService.unmarkTableDeleted.mock.calls.map((c: any[]) => c[0].table_name).sort();
|
||||
expect(unmarked).toEqual(['clientes', 'pedidos']);
|
||||
});
|
||||
|
||||
it('rolls back the marks made so far if a later mark fails (atomic)', async () => {
|
||||
// second mark fails → first must be rolled back, no platform delete attempted
|
||||
inputsService.markTableDeleted
|
||||
.mockResolvedValueOnce({ is_deleted: true, deleted_at: 't' })
|
||||
.mockRejectedValueOnce(new Error('dynamo boom'));
|
||||
|
||||
await expect(
|
||||
controller.deleteTables('pid', 'iid', { table_names: ['pedidos', 'clientes'] }, mockUser),
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(inputsService.unmarkTableDeleted).toHaveBeenCalledTimes(1);
|
||||
expect(inputsService.unmarkTableDeleted.mock.calls[0][0].table_name).toBe('pedidos');
|
||||
// never reached the platform delete
|
||||
const attemptedDelete = platformApiService.proxy.mock.calls.some(([m]: any[]) => m === 'DELETE');
|
||||
expect(attemptedDelete).toBe(false);
|
||||
});
|
||||
|
||||
it('404s when a requested table has no matching job', async () => {
|
||||
platformApiService.proxy.mockImplementation((method: string) =>
|
||||
Promise.resolve(method === 'GET' ? pipelineWithJobs() : {}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
controller.deleteTables('pid', 'iid', { table_names: ['pedidos', 'ghost'] }, mockUser),
|
||||
).rejects.toThrow();
|
||||
// the successful mark (pedidos) must be rolled back
|
||||
expect(inputsService.unmarkTableDeleted).toHaveBeenCalled();
|
||||
tables.addTable.mockResolvedValue({ job_ids: ['p_2'], skipped: [] });
|
||||
await expect(controller.addTable('pid', 'iid', body, mockUser)).resolves.toEqual({ job_ids: ['p_2'], skipped: [] });
|
||||
expect(tables.addTable).toHaveBeenCalledWith('pid', 'iid', body, mockUser);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
Inject,
|
||||
BadRequestException,
|
||||
HttpException,
|
||||
NotFoundException,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
|
||||
@@ -31,10 +30,9 @@ import { CustomersService } from '../customers/customers.service';
|
||||
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
|
||||
import { CatalogService } from '../catalog/catalog.service';
|
||||
import { PackTheMetadata } from '../../utils/PackTheMetadata';
|
||||
import { ValidationTableDTO } from './platform-api.dto';
|
||||
import { AddCdcTableBody, DeleteTableBody, DeleteTablesBody, ValidationTableDTO } from './platform-api.dto';
|
||||
import { InputsService } from '../inputs/inputs.service';
|
||||
import { toCdcTable } from '../inputs/cdc-table.mapper';
|
||||
import { PipelinesService } from '../pipelinesV2/pipelines.service';
|
||||
import { PipelineTablesService } from './pipeline-tables.service';
|
||||
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
|
||||
|
||||
|
||||
@@ -63,7 +61,7 @@ export class PlatformApiController {
|
||||
private readonly customersService: CustomersService,
|
||||
private readonly catalogService: CatalogService,
|
||||
private readonly inputsService: InputsService,
|
||||
private readonly pipelinesClientService: PipelinesService,
|
||||
private readonly pipelineTablesService: PipelineTablesService,
|
||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||
) {
|
||||
this.logger = dadosferaLogger.logger;
|
||||
@@ -989,60 +987,16 @@ export class PlatformApiController {
|
||||
}
|
||||
|
||||
@Delete('pipelines/:pipelineId/inputs/:inputId')
|
||||
@ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' })
|
||||
@ApiOperation({ summary: 'Mark a table as deleted and remove its job via platform-api' })
|
||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
||||
@UseGuards(PipelineExecutionGuard)
|
||||
async deleteTable(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('inputId') inputId: string,
|
||||
@Body() body: { table_name: string },
|
||||
@Body() body: DeleteTableBody,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const tableName = body.table_name;
|
||||
const info = {
|
||||
customer_id: user.customer_id,
|
||||
customer: user.customer_name,
|
||||
user_id: user.user_id,
|
||||
};
|
||||
|
||||
this.logger.info('deleteTable: marking table as deleted', { inputId, tableName });
|
||||
const updatedInput: any = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info });
|
||||
this.logger.info('deleteTable: table marked as deleted', { inputId, tableName });
|
||||
|
||||
try {
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
this.logger.info('deleteTable: fetching pipeline from platform-api', { pipelineId, normalizedPipelineId });
|
||||
const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
|
||||
this.logger.info('deleteTable: pipeline fetched', { jobCount: platformPipeline?.jobs?.length });
|
||||
|
||||
const job = platformPipeline?.jobs?.find((j: any) => j.input?.table_name === tableName);
|
||||
if (!job) throw new NotFoundException(`Job for table '${tableName}' not found in pipeline`);
|
||||
|
||||
this.logger.info('deleteTable: deleting job from platform-api', { jobId: job.job_id });
|
||||
if (job.input?.connector === 'cdc') {
|
||||
// CDC: reconfigure the Debezium/Kafka-Connect connector (stop
|
||||
// replicating this table); keep the landed Snowflake data.
|
||||
await this.platformApiService.proxy(
|
||||
'DELETE',
|
||||
`/pipeline/${normalizedPipelineId}/jobs`,
|
||||
user,
|
||||
{ job_ids: [job.job_id], delete_snowflake_tables: false },
|
||||
);
|
||||
} else {
|
||||
await this.platformApiService.proxy('DELETE', `/jobs/${job.job_id}`, user);
|
||||
}
|
||||
this.logger.info('deleteTable: job deleted', { jobId: job.job_id });
|
||||
|
||||
return { name: tableName, is_deleted: updatedInput.is_deleted ?? true, deleted_at: updatedInput.deleted_at };
|
||||
} catch (error) {
|
||||
this.logger.error('deleteTable: platform-api delete failed, attempting rollback', { tableName, error: error.message });
|
||||
try {
|
||||
await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: tableName, info });
|
||||
} catch (rollbackError) {
|
||||
this.logger.error('deleteTable: rollback failed', { tableName, error: rollbackError.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return this.pipelineTablesService.removeTable(pipelineId, inputId, body.table_name, user);
|
||||
}
|
||||
|
||||
@Delete('pipelines/:pipelineId/inputs/:inputId/tables')
|
||||
@@ -1052,66 +1006,14 @@ export class PlatformApiController {
|
||||
async deleteTables(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('inputId') inputId: string,
|
||||
@Body() body: { table_names: string[] },
|
||||
@Body() body: DeleteTablesBody,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const tableNames = body.table_names ?? [];
|
||||
if (tableNames.length === 0) {
|
||||
throw new BadRequestException('table_names must be a non-empty array');
|
||||
}
|
||||
const info = {
|
||||
customer_id: user.customer_id,
|
||||
customer: user.customer_name,
|
||||
user_id: user.user_id,
|
||||
};
|
||||
|
||||
// 1) Soft-delete each table in DynamoDB, tracking which succeeded so a later
|
||||
// failure only rolls back the marks made in THIS call.
|
||||
const marked: string[] = [];
|
||||
const rollback = async () => {
|
||||
for (const name of marked) {
|
||||
try {
|
||||
await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: name, info });
|
||||
} catch (rollbackError) {
|
||||
this.logger.error('deleteTables: rollback failed', { tableName: name, error: rollbackError.message });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
for (const name of tableNames) {
|
||||
await this.inputsService.markTableDeleted({ input_id: inputId, table_name: name, info });
|
||||
marked.push(name);
|
||||
}
|
||||
|
||||
// 2) Resolve all table_names -> job_ids from the platform pipeline (one GET).
|
||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||
const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
|
||||
const jobs = platformPipeline?.jobs ?? [];
|
||||
|
||||
const jobIds: string[] = [];
|
||||
for (const name of tableNames) {
|
||||
const job = jobs.find((j: any) => j.input?.table_name === name);
|
||||
if (!job) throw new NotFoundException(`Job for table '${name}' not found in pipeline`);
|
||||
jobIds.push(job.job_id);
|
||||
}
|
||||
|
||||
// 3) Remove them all in ONE platform call so the Debezium source + Snowflake
|
||||
// sink connectors are reconfigured a single time, not once per table.
|
||||
await this.platformApiService.proxy(
|
||||
'DELETE',
|
||||
`/pipeline/${normalizedPipelineId}/jobs`,
|
||||
user,
|
||||
{ job_ids: jobIds, delete_snowflake_tables: false },
|
||||
);
|
||||
this.logger.info('deleteTables: jobs deleted', { jobIds });
|
||||
|
||||
return { table_names: tableNames, deleted: true };
|
||||
} catch (error) {
|
||||
this.logger.error('deleteTables: failed, rolling back this call\'s marks', { tableNames, error: error.message });
|
||||
await rollback();
|
||||
throw error;
|
||||
}
|
||||
return this.pipelineTablesService.removeTables(pipelineId, inputId, tableNames, user);
|
||||
}
|
||||
|
||||
@Post('pipelines/:pipelineId/inputs/:inputId/tables')
|
||||
@@ -1121,68 +1023,10 @@ export class PlatformApiController {
|
||||
async addTable(
|
||||
@Param('pipelineId') pipelineId: string,
|
||||
@Param('inputId') inputId: string,
|
||||
@Body() body: {
|
||||
table_name: string;
|
||||
table_schema: string;
|
||||
primary_keys: string[];
|
||||
destinations: {
|
||||
raw: { table_schema: string; table_name: string };
|
||||
qualify: { table_schema: string; table_name: string };
|
||||
};
|
||||
// Iceberg destination only (protospack CdcTable.iceberg_table_name);
|
||||
// absent for snowflake, back-compat.
|
||||
iceberg_table_name?: string;
|
||||
// Per-table deduped (qualify) Iceberg table name (protospack
|
||||
// CdcTable.iceberg_qualify_table_name); absent => same as the raw name.
|
||||
iceberg_qualify_table_name?: string;
|
||||
// Source column schema for iceberg deduped table pre-create (protospack CdcTable.columns).
|
||||
columns?: { name: string; type: string; is_primary_key: boolean }[];
|
||||
// Columns the user chose to ignore -> Debezium column.exclude.list.
|
||||
column_exclude_list?: string[];
|
||||
},
|
||||
@Body() body: AddCdcTableBody,
|
||||
@User() user: RequestUser,
|
||||
) {
|
||||
const info = {
|
||||
customer_id: user.customer_id,
|
||||
customer: user.customer_name,
|
||||
user_id: user.user_id,
|
||||
};
|
||||
|
||||
const cdcTable = toCdcTable(body);
|
||||
|
||||
this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });
|
||||
await this.inputsService.addCdcTable({ client_id: user.customer_id, id: inputId, table: cdcTable, info });
|
||||
this.logger.info('addTable: DynamoDB row appended', { inputId, tableName: body.table_name });
|
||||
|
||||
try {
|
||||
const customInfo = {
|
||||
customer_id: user.customer_id,
|
||||
user_id: user.user_id,
|
||||
customer: user.customer_name,
|
||||
};
|
||||
|
||||
const res = await this.pipelinesClientService.addCdcJobs({
|
||||
pipeline_id: pipelineId,
|
||||
input_id: inputId,
|
||||
tables: [{
|
||||
table_schema: body.table_schema,
|
||||
table_name: body.table_name,
|
||||
primary_keys: body.primary_keys,
|
||||
destinations: body.destinations,
|
||||
}],
|
||||
info: customInfo,
|
||||
});
|
||||
|
||||
return res;
|
||||
} catch (error) {
|
||||
this.logger.error('addTable: platform AddJobs failed, rolling back the DynamoDB row', { tableName: body.table_name, error: error.message });
|
||||
try {
|
||||
await this.inputsService.removeCdcTable({ client_id: user.customer_id, id: inputId, table_name: body.table_name, info });
|
||||
} catch (rbErr) {
|
||||
this.logger.error('addTable: rollback failed', { error: rbErr.message });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return this.pipelineTablesService.addTable(pipelineId, inputId, body, user);
|
||||
}
|
||||
|
||||
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
|
||||
|
||||
@@ -1,9 +1,66 @@
|
||||
import { ApiProperty } from "@nestjs/swagger";
|
||||
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
|
||||
|
||||
export class ValidationTableDTO {
|
||||
@ApiProperty()
|
||||
tables: Array<{
|
||||
table_name: string;
|
||||
table_schema: string;
|
||||
}>
|
||||
}
|
||||
@ApiProperty()
|
||||
tables: Array<{
|
||||
table_name: string;
|
||||
table_schema: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export class DeleteTableBody {
|
||||
@ApiProperty()
|
||||
table_name: string;
|
||||
}
|
||||
|
||||
export class DeleteTablesBody {
|
||||
@ApiProperty({ type: [String] })
|
||||
table_names: string[];
|
||||
}
|
||||
|
||||
export class CdcTableDestination {
|
||||
@ApiProperty()
|
||||
table_schema: string;
|
||||
@ApiProperty()
|
||||
table_name: string;
|
||||
}
|
||||
|
||||
export class CdcTableDestinations {
|
||||
@ApiProperty({ type: CdcTableDestination })
|
||||
raw: CdcTableDestination;
|
||||
@ApiProperty({ type: CdcTableDestination })
|
||||
qualify: CdcTableDestination;
|
||||
}
|
||||
|
||||
export class CdcColumnBody {
|
||||
@ApiProperty()
|
||||
name: string;
|
||||
@ApiProperty()
|
||||
type: string;
|
||||
@ApiProperty()
|
||||
is_primary_key: boolean;
|
||||
}
|
||||
|
||||
export class AddCdcTableBody {
|
||||
@ApiProperty()
|
||||
table_name: string;
|
||||
@ApiProperty()
|
||||
table_schema: string;
|
||||
@ApiProperty({ type: [String] })
|
||||
primary_keys: string[];
|
||||
@ApiProperty({ type: CdcTableDestinations })
|
||||
destinations: CdcTableDestinations;
|
||||
// Iceberg destination only (protospack CdcTable.iceberg_table_name);
|
||||
// absent for snowflake, back-compat.
|
||||
@ApiPropertyOptional()
|
||||
iceberg_table_name?: string;
|
||||
// Per-table deduped (qualify) Iceberg table name; absent => same as raw.
|
||||
@ApiPropertyOptional()
|
||||
iceberg_qualify_table_name?: string;
|
||||
// Source column schema for iceberg deduped table pre-create.
|
||||
@ApiPropertyOptional({ type: [CdcColumnBody] })
|
||||
columns?: CdcColumnBody[];
|
||||
// Columns the user chose to ignore -> Debezium column.exclude.list.
|
||||
@ApiPropertyOptional({ type: [String] })
|
||||
column_exclude_list?: string[];
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||
|
||||
import { PlatformApiController } from './platform-api.controller';
|
||||
import { PlatformApiService } from './platform-api.service';
|
||||
import { PipelineTablesService } from './pipeline-tables.service';
|
||||
import { ElasticsearchModule } from '../../services/elasticsearch';
|
||||
import { DynamoDBModule } from '../../services/dynamodb';
|
||||
import { CustomersModule } from '../customers/customers.module';
|
||||
@@ -21,7 +22,7 @@ import { PipelinesV2Module } from '../pipelinesV2/pipelines.module';
|
||||
forwardRef(() => PipelinesV2Module),
|
||||
],
|
||||
controllers: [PlatformApiController],
|
||||
providers: [PlatformApiService, DadosferaLogger],
|
||||
providers: [PlatformApiService, PipelineTablesService, DadosferaLogger],
|
||||
exports: [PlatformApiService],
|
||||
})
|
||||
export class PlatformApiModule {}
|
||||
|
||||
Reference in New Issue
Block a user