mirror of
https://github.com/dadosfera/maestro.git
synced 2026-11-22 18:49:07 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e06e7498db | ||
|
|
922a34a393 | ||
|
|
a795f543f6 | ||
|
|
15b5c49fb2 | ||
|
|
1730ec2753 | ||
|
|
dc4609cbc5 |
@@ -2,21 +2,9 @@ name: Test
|
|||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- beta
|
|
||||||
- main
|
- main
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# Blocks a local (file:/tarball/overlay) protospack-v2 dependency from
|
|
||||||
# reaching staging (beta) or prod (main).
|
|
||||||
protospack-dep-guard:
|
|
||||||
if: github.base_ref == 'beta' || github.base_ref == 'main'
|
|
||||||
runs-on: [self-hosted, prd]
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
- name: Check protospack-v2 is consumed from the registry
|
|
||||||
run: node scripts/check-protospack-dep.js
|
|
||||||
|
|
||||||
test:
|
test:
|
||||||
runs-on: [self-hosted, prd]
|
runs-on: [self-hosted, prd]
|
||||||
env:
|
env:
|
||||||
|
|||||||
@@ -1,56 +0,0 @@
|
|||||||
# maestro — development policy
|
|
||||||
|
|
||||||
Maestro is the NestJS BFF between the Angular frontend and the gRPC services
|
|
||||||
(pi-factory, in-factory) / platform-api. These rules come from code review and
|
|
||||||
apply to every change; the same rules live in pi-factory and in-factory.
|
|
||||||
|
|
||||||
## Layering
|
|
||||||
- Controllers are thin: decorators, body validation, one call into a service,
|
|
||||||
response shape. Orchestration (multi-step calls, rollbacks, platform-api or
|
|
||||||
gRPC round-trips) lives in a `*.service.ts`. Example:
|
|
||||||
`platform-api/pipeline-tables.service.ts`.
|
|
||||||
- One platform-api route per operation. If the platform already dispatches by
|
|
||||||
pipeline type (batch vs CDC), do not branch on the type here — call the
|
|
||||||
route that dispatches (e.g. `DELETE /pipeline/{id}/jobs` for table removal,
|
|
||||||
never `DELETE /jobs/{id}` from maestro).
|
|
||||||
|
|
||||||
## Types
|
|
||||||
- No `any` / `as any`. gRPC calls take the protospack request type exactly
|
|
||||||
(`AddCdcTableRequest`, `MarkTableDeletedRequest`, ...); DTOs and interfaces
|
|
||||||
are explicit classes/interfaces. Use `Record<string, T>` or `unknown` (with
|
|
||||||
narrowing) when a shape is genuinely open — never `any`.
|
|
||||||
- Protospack payloads are built in one mapper per entity (e.g.
|
|
||||||
`inputs/cdc-table.mapper.ts`), so a field that mirrors another
|
|
||||||
(`CdcTable.name` == `table_name`) is derived in exactly one place.
|
|
||||||
|
|
||||||
## Pipeline type
|
|
||||||
- Decide CDC vs batch with `src/utils/cdc.ts` (`isCdcPlugin`, `isCdcJob`,
|
|
||||||
`isCdcPipeline`), never with inline `plugin.endsWith('_cdc')`, `!!x`, or
|
|
||||||
the absence of some other data (e.g. "no run history ⇒ editable").
|
|
||||||
- The platform-api stamps `job.input.connector === 'cdc'` on CDC jobs; that
|
|
||||||
is the authoritative discriminator once a pipeline exists.
|
|
||||||
|
|
||||||
## Dependencies
|
|
||||||
- `@dadosfera/protospack-v2` is consumed from CodeArtifact, pinned to an exact
|
|
||||||
version (`npm i @dadosfera/protospack-v2@<version> --save-exact`). A `file:`
|
|
||||||
/ tarball reference is for local development only and must never be
|
|
||||||
committed. Note: `^3.40.0-beta.N` resolves to the stable `3.40.0` —
|
|
||||||
prereleases must be pinned exactly.
|
|
||||||
|
|
||||||
## Commits and releases
|
|
||||||
- Deploys are cut by semantic-release with the eslint preset: the commit
|
|
||||||
title MUST start with `FIX:` (patch), `UPDATE:` or `FEAT:` (minor). A
|
|
||||||
lowercase `feat(scope): ...` merges without producing a version, so the
|
|
||||||
code never reaches stg/prd.
|
|
||||||
- Pushing to `beta` deploys stg; `main` deploys prd.
|
|
||||||
|
|
||||||
## Tests
|
|
||||||
- Every service method with a rollback path has a spec covering the
|
|
||||||
happy path, the platform failure (rollback fires) and a rollback failure
|
|
||||||
(does not mask the original error). Run `npx jest <path>` for a folder,
|
|
||||||
`npx tsc --noEmit -p tsconfig.json` for types.
|
|
||||||
- Several gRPC client configs read `process.env` at import time, so the full
|
|
||||||
suite needs the service URLs set (any `0.0.0.0:<port>` value works):
|
|
||||||
`DUC_URL=0.0.0.0:50051 INFACTORY_URL=0.0.0.0:50052 PIFACTORY_URL=0.0.0.0:50053 npx jest`.
|
|
||||||
A `Cannot read properties of undefined (reading 'startsWith')` at import is
|
|
||||||
this, not a broken test.
|
|
||||||
@@ -29,7 +29,6 @@ COPY . .
|
|||||||
# unit test specific build
|
# unit test specific build
|
||||||
FROM ci_image AS test
|
FROM ci_image AS test
|
||||||
ENV DUC_URL=0.0.0.0:50051
|
ENV DUC_URL=0.0.0.0:50051
|
||||||
ENV INFACTORY_URL=0.0.0.0:50052
|
|
||||||
ENTRYPOINT ["npm", "run", "test"]
|
ENTRYPOINT ["npm", "run", "test"]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,759 @@
|
|||||||
|
# Orchest Module Identity Implementation Plan
|
||||||
|
|
||||||
|
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||||
|
|
||||||
|
**Goal:** Add a Maestro endpoint `GET /auth/module-identity` that translates a Dadosfera session into the `X-Auth-*` identity headers Orchest's RBAC trusts, gating on a module permission and (for service ingresses) authorizing per-project against orchest-api.
|
||||||
|
|
||||||
|
**Architecture:** One route serves two nginx `auth_request` callers, keyed on whether the ingress annotation carries authz query params. Phase 1: authenticate the `ddf-auth` cookie via Maestro's existing JWKS verify, gate on the module permission (seqid 31), emit identity headers, set `X-Auth-Roles: admin` for Super Admin (seqid 34). Phase 2: when the annotation carries `permission`+`project_uuid`, additionally call the calling tenant's orchest-api `/api/authz/check` (host derived from the JWT's `customer_name` + the `orchest-{module}-{customer_name}` namespace convention) and relay allow/deny, fail-closed. A small orchest-side change flips `service_access_auth_url` to build a scoped URL in Maestro mode.
|
||||||
|
|
||||||
|
**Tech Stack:** NestJS (controllers/providers, Jest via `Test.createTestingModule`), Express `Request`/`Response`, `jsonwebtoken`, plain `process.env` config. Orchest side: Python (`lib/python/orchest-internals`).
|
||||||
|
|
||||||
|
**Spec:** `dbt-to-orchest` repo — `docs/superpowers/specs/2026-08-20-maestro-module-identity-design.md` (the Maestro repo does not hold the spec; executors read it there).
|
||||||
|
|
||||||
|
## Global Constraints
|
||||||
|
|
||||||
|
- **Config via `process.env`** — Maestro reads env directly (no ConfigService). Pattern: `const X = process.env.X || '<default>'` (see `authentication.guard.ts:140`).
|
||||||
|
- **Config values (verbatim):** `ORCHEST_MODULE_PERMISSION_SEQID` default `31` (Intelligence/"Orchest Module", `intelligence:open`); `ORCHEST_ADMIN_PERMISSION_SEQIDS` default `34` (Super Admin, `users:admin`), a comma-separated list parsed to numbers; `ORCHEST_NAMESPACE_MODULE` default `intelli` (∈ `intelli`|`process`).
|
||||||
|
- **Token source is the `ddf-auth` COOKIE**, not the `Authorization` header (the nginx `auth_request` subrequest carries the browser cookie). Never read `Authorization` in this route.
|
||||||
|
- **`permission`/`project_uuid` come from `request.query`** (the orchest-api-authored annotation), never from the end-user URL. The end-user URL is `X-Original-URI` and MUST NOT be read for authz.
|
||||||
|
- **Fail-closed:** any error reaching orchest-api, or an unexpected status, → deny (never allow).
|
||||||
|
- **`/auth/me` is not modified.** Other services depend on it.
|
||||||
|
- **No orchest-api code change.** `/api/authz/check`, the `AuthCustom/AuthUrl` ingress path, and RBAC already exist.
|
||||||
|
- **Namespace pattern:** `orchest-{module}-{customer_name}`. `dadosferademo2` is the one exception (being removed) — explicitly unsupported, no special-casing.
|
||||||
|
- Branch: `feat/orchest-module-identity` (off `origin/beta`), already created.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## File Structure
|
||||||
|
|
||||||
|
**Maestro (`feat/orchest-module-identity` off beta):**
|
||||||
|
- Modify `src/modules/auth/auth.service.ts` — make `validateJwtToken` public (or add a public `verifyAccessToken` wrapper); add `authorizeOrchestServiceAccess(...)` helper (Phase 2).
|
||||||
|
- Modify `src/modules/auth/auth.controller.ts` — add the `GET /auth/module-identity` route.
|
||||||
|
- Create `src/modules/auth/orchest-identity.ts` — pure, testable helpers: `parseAdminSeqids(env)`, `isModuleAllowed(perms, gateSeqid)`, `isAdmin(perms, adminSeqids)`, `tenantOrchestApiHost(customerName, module)`, `isAllowedOrchestApiHost(host)`. Keeps set-membership/string logic out of the controller so it unit-tests without HTTP.
|
||||||
|
- Create `src/modules/auth/orchest-identity.spec.ts` — unit tests for the helpers.
|
||||||
|
- Create `src/modules/auth/module-identity.controller.spec.ts` — controller tests (mock `AuthClientService`, fake `Request`/`Response`).
|
||||||
|
|
||||||
|
**dbt-to-orchest repo (Phase 2 orchest-side, separate branch there):**
|
||||||
|
- Modify `lib/python/orchest-internals/_orchest/internals/utils.py:19-51` — `service_access_auth_url` Maestro branch.
|
||||||
|
- Modify `lib/python/orchest-internals/tests/…` (or wherever `utils` is tested) — add the Maestro-mode case.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PHASE 1 — Identity (webserver ingress)
|
||||||
|
|
||||||
|
### Task 1: Pure helpers for permission mapping
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Create: `src/modules/auth/orchest-identity.ts`
|
||||||
|
- Test: `src/modules/auth/orchest-identity.spec.ts`
|
||||||
|
|
||||||
|
**Interfaces:**
|
||||||
|
- Consumes: nothing.
|
||||||
|
- Produces:
|
||||||
|
- `parseAdminSeqids(raw: string | undefined): number[]` — parse `"34"` / `"34,40"` → `[34]` / `[34,40]`; empty/undefined → `[34]`.
|
||||||
|
- `moduleGateSeqid(raw: string | undefined): number` — parse `ORCHEST_MODULE_PERMISSION_SEQID` → number; default `31`.
|
||||||
|
- `isModuleAllowed(perms: number[], gateSeqid: number): boolean`
|
||||||
|
- `isAdmin(perms: number[], adminSeqids: number[]): boolean`
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the failing test**
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import {
|
||||||
|
parseAdminSeqids,
|
||||||
|
moduleGateSeqid,
|
||||||
|
isModuleAllowed,
|
||||||
|
isAdmin,
|
||||||
|
} from './orchest-identity';
|
||||||
|
|
||||||
|
describe('orchest-identity mapping', () => {
|
||||||
|
it('parseAdminSeqids: default, single, list, whitespace', () => {
|
||||||
|
expect(parseAdminSeqids(undefined)).toEqual([34]);
|
||||||
|
expect(parseAdminSeqids('')).toEqual([34]);
|
||||||
|
expect(parseAdminSeqids('34')).toEqual([34]);
|
||||||
|
expect(parseAdminSeqids('34,40')).toEqual([34, 40]);
|
||||||
|
expect(parseAdminSeqids(' 34 , 40 ')).toEqual([34, 40]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('moduleGateSeqid: default and override', () => {
|
||||||
|
expect(moduleGateSeqid(undefined)).toBe(31);
|
||||||
|
expect(moduleGateSeqid('43')).toBe(43);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('isModuleAllowed', () => {
|
||||||
|
expect(isModuleAllowed([31, 5], 31)).toBe(true);
|
||||||
|
expect(isModuleAllowed([5, 7], 31)).toBe(false);
|
||||||
|
expect(isModuleAllowed([], 31)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('isAdmin: intersection', () => {
|
||||||
|
expect(isAdmin([31, 34], [34])).toBe(true);
|
||||||
|
expect(isAdmin([31], [34])).toBe(false);
|
||||||
|
expect(isAdmin([99], [34, 99])).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run test to verify it fails**
|
||||||
|
|
||||||
|
Run: `npx jest src/modules/auth/orchest-identity.spec.ts -t 'orchest-identity mapping'`
|
||||||
|
Expected: FAIL — `Cannot find module './orchest-identity'`.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Write minimal implementation**
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// src/modules/auth/orchest-identity.ts
|
||||||
|
export function parseAdminSeqids(raw: string | undefined): number[] {
|
||||||
|
if (!raw || !raw.trim()) return [34];
|
||||||
|
return raw
|
||||||
|
.split(',')
|
||||||
|
.map((s) => Number(s.trim()))
|
||||||
|
.filter((n) => Number.isInteger(n));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function moduleGateSeqid(raw: string | undefined): number {
|
||||||
|
const n = Number(raw);
|
||||||
|
return Number.isInteger(n) && n > 0 ? n : 31;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isModuleAllowed(perms: number[], gateSeqid: number): boolean {
|
||||||
|
return Array.isArray(perms) && perms.includes(gateSeqid);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isAdmin(perms: number[], adminSeqids: number[]): boolean {
|
||||||
|
return (
|
||||||
|
Array.isArray(perms) && perms.some((p) => adminSeqids.includes(p))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run test to verify it passes**
|
||||||
|
|
||||||
|
Run: `npx jest src/modules/auth/orchest-identity.spec.ts -t 'orchest-identity mapping'`
|
||||||
|
Expected: PASS.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/modules/auth/orchest-identity.ts src/modules/auth/orchest-identity.spec.ts
|
||||||
|
git commit -m "feat(orchest-identity): permission-mapping helpers"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 2: Expose JWKS verify on AuthClientService
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/modules/auth/auth.service.ts:413-425`
|
||||||
|
|
||||||
|
**Interfaces:**
|
||||||
|
- Consumes: existing `getPublicKeys()`.
|
||||||
|
- Produces: `public async verifyAccessToken(token: string): Promise<any>` — verifies the JWT against JWKS and returns its payload; throws on missing/invalid token or unknown `kid`. (Rename of the existing private `validateJwtToken`, kept callable by `validateUserSession`.)
|
||||||
|
|
||||||
|
- [ ] **Step 1: Make the method public and rename**
|
||||||
|
|
||||||
|
The method already does exactly the needed decode+verify. Rename `validateJwtToken` → `verifyAccessToken`, change `private` → `public`, and update its one caller.
|
||||||
|
|
||||||
|
In `src/modules/auth/auth.service.ts`, change line 413:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
public async verifyAccessToken(token: string) {
|
||||||
|
const decoded: any = token && jwt.decode(token, { complete: true });
|
||||||
|
if (!decoded) throw new Error('Invalid token');
|
||||||
|
|
||||||
|
const { kid } = decoded.header;
|
||||||
|
const { keys } = await this.getPublicKeys();
|
||||||
|
const pemValue = keys.find((k) => k.kid === kid)?.pem;
|
||||||
|
if (!pemValue) throw new Error('Public key not found');
|
||||||
|
jwt.verify(token, pemValue);
|
||||||
|
|
||||||
|
return decoded.payload;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
And update the caller in `validateUserSession` (was line 310):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const payload = await this.verifyAccessToken(accessToken);
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Verify existing suite still compiles/passes for auth.service**
|
||||||
|
|
||||||
|
Run: `npx jest src/modules/auth`
|
||||||
|
Expected: PASS (no behavior change; `/auth/me` path unaffected). If there is no existing auth.service spec, run `npx tsc --noEmit` to confirm the rename compiles.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/modules/auth/auth.service.ts
|
||||||
|
git commit -m "refactor(auth): expose verifyAccessToken (was private validateJwtToken)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 3: `GET /auth/module-identity` — identity + module gate
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/modules/auth/auth.controller.ts` (add route beside `getMe`, ~after line 533)
|
||||||
|
- Test: `src/modules/auth/module-identity.controller.spec.ts`
|
||||||
|
|
||||||
|
**Interfaces:**
|
||||||
|
- Consumes: `AuthClientService.verifyAccessToken` (Task 2); `parseAdminSeqids`/`moduleGateSeqid`/`isModuleAllowed`/`isAdmin` (Task 1).
|
||||||
|
- Produces: route `GET /auth/module-identity`. On 200 sets response headers `X-Auth-User`, `X-Auth-Username`, and (admin only) `X-Auth-Roles: admin`; empty body. 401 (no/invalid cookie), 403 (lacks module gate).
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the failing test**
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { Test } from '@nestjs/testing';
|
||||||
|
import { AuthController } from './auth.controller';
|
||||||
|
import { AuthClientService } from './auth.service';
|
||||||
|
|
||||||
|
function res() {
|
||||||
|
const headers: Record<string, string> = {};
|
||||||
|
const r: any = {
|
||||||
|
_status: 0,
|
||||||
|
_sent: undefined,
|
||||||
|
set: (k: string, v: string) => { headers[k] = v; return r; },
|
||||||
|
status: (c: number) => { r._status = c; return r; },
|
||||||
|
send: (b?: any) => { r._sent = b ?? ''; return r; },
|
||||||
|
json: (b?: any) => { r._sent = b; return r; },
|
||||||
|
_headers: headers,
|
||||||
|
};
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
function req(cookie?: string, query: Record<string, string> = {}) {
|
||||||
|
return { cookies: cookie ? { 'ddf-auth': cookie } : {}, query } as any;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('GET /auth/module-identity — identity', () => {
|
||||||
|
let controller: AuthController;
|
||||||
|
const auth = { verifyAccessToken: jest.fn() } as unknown as AuthClientService;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
jest.resetAllMocks();
|
||||||
|
process.env.ORCHEST_MODULE_PERMISSION_SEQID = '31';
|
||||||
|
process.env.ORCHEST_ADMIN_PERMISSION_SEQIDS = '34';
|
||||||
|
const mod = await Test.createTestingModule({
|
||||||
|
controllers: [AuthController],
|
||||||
|
providers: [{ provide: AuthClientService, useValue: auth }],
|
||||||
|
})
|
||||||
|
// Any other providers AuthController injects must be stubbed here the
|
||||||
|
// same way (ApiKeyService, DadosferaLogger, etc.). Add them as the
|
||||||
|
// compile step reports missing providers.
|
||||||
|
.compile();
|
||||||
|
controller = mod.get(AuthController);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('no cookie → 401', async () => {
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req(undefined), r);
|
||||||
|
expect(r._status).toBe(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('valid + module + admin → 200 with X-Auth-Roles: admin', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-1', username: 'alice', permissions: [31, 34],
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._status).toBe(200);
|
||||||
|
expect(r._headers['X-Auth-User']).toBe('u-1');
|
||||||
|
expect(r._headers['X-Auth-Username']).toBe('alice');
|
||||||
|
expect(r._headers['X-Auth-Roles']).toBe('admin');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('valid + module, not admin → 200, no X-Auth-Roles', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-2', username: 'bob', permissions: [31],
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._status).toBe(200);
|
||||||
|
expect(r._headers['X-Auth-Roles']).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('valid, lacks module → 403', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-3', username: 'carol', permissions: [5],
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._status).toBe(403);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('verify throws (expired/bad) → 401', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockRejectedValue(new Error('bad'));
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._status).toBe(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('admin seqids extended by config → 200 admin', async () => {
|
||||||
|
process.env.ORCHEST_ADMIN_PERMISSION_SEQIDS = '34,99';
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-4', username: 'dana', permissions: [31, 99],
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._headers['X-Auth-Roles']).toBe('admin');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run test to verify it fails**
|
||||||
|
|
||||||
|
Run: `npx jest src/modules/auth/module-identity.controller.spec.ts`
|
||||||
|
Expected: FAIL — `controller.moduleIdentity is not a function` (and possibly missing-provider errors, which tell you which providers to stub — add them to the `providers` array per the comment).
|
||||||
|
|
||||||
|
- [ ] **Step 3: Write minimal implementation**
|
||||||
|
|
||||||
|
Add to `auth.controller.ts` (import the helpers at top; `AuthClientService` is already injected as `this.authClient`):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import {
|
||||||
|
parseAdminSeqids,
|
||||||
|
moduleGateSeqid,
|
||||||
|
isModuleAllowed,
|
||||||
|
isAdmin,
|
||||||
|
} from './orchest-identity';
|
||||||
|
```
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
@Get('module-identity')
|
||||||
|
async moduleIdentity(@Req() req: Request, @Res() res: Response) {
|
||||||
|
const token = req.cookies?.['ddf-auth'];
|
||||||
|
if (!token) {
|
||||||
|
return res.status(401).send();
|
||||||
|
}
|
||||||
|
|
||||||
|
let payload: any;
|
||||||
|
try {
|
||||||
|
payload = await this.authClient.verifyAccessToken(token);
|
||||||
|
} catch (e) {
|
||||||
|
return res.status(401).send();
|
||||||
|
}
|
||||||
|
|
||||||
|
const perms: number[] = payload?.permissions ?? [];
|
||||||
|
const gate = moduleGateSeqid(process.env.ORCHEST_MODULE_PERMISSION_SEQID);
|
||||||
|
if (!isModuleAllowed(perms, gate)) {
|
||||||
|
return res.status(403).send();
|
||||||
|
}
|
||||||
|
|
||||||
|
res.set('X-Auth-User', String(payload.user_id));
|
||||||
|
res.set('X-Auth-Username', String(payload.username ?? ''));
|
||||||
|
if (isAdmin(perms, parseAdminSeqids(process.env.ORCHEST_ADMIN_PERMISSION_SEQIDS))) {
|
||||||
|
res.set('X-Auth-Roles', 'admin');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 2 authz branch is inserted here (Task 5) before the 200.
|
||||||
|
return res.status(200).send();
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run test to verify it passes**
|
||||||
|
|
||||||
|
Run: `npx jest src/modules/auth/module-identity.controller.spec.ts`
|
||||||
|
Expected: PASS (all identity cases).
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/modules/auth/auth.controller.ts src/modules/auth/module-identity.controller.spec.ts
|
||||||
|
git commit -m "feat(auth): GET /auth/module-identity — identity + module gate"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PHASE 2 — Per-service authz
|
||||||
|
|
||||||
|
### Task 4: Tenant orchest-api host derivation + allowlist
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/modules/auth/orchest-identity.ts`
|
||||||
|
- Modify: `src/modules/auth/orchest-identity.spec.ts`
|
||||||
|
|
||||||
|
**Interfaces:**
|
||||||
|
- Consumes: nothing.
|
||||||
|
- Produces:
|
||||||
|
- `tenantOrchestApiHost(customerName: string, module: string): string` — returns `orchest-api.orchest-{module}-{customerName}.svc.cluster.local`.
|
||||||
|
- `isAllowedOrchestApiHost(host: string): boolean` — matches `^orchest-api\.orchest-(intelli|process)-[a-z0-9-]+\.svc\.cluster\.local$`.
|
||||||
|
- `namespaceModule(raw: string | undefined): 'intelli' | 'process'` — parse `ORCHEST_NAMESPACE_MODULE`; default `intelli`; anything not `process` → `intelli`.
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the failing test (append to orchest-identity.spec.ts)**
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import {
|
||||||
|
tenantOrchestApiHost,
|
||||||
|
isAllowedOrchestApiHost,
|
||||||
|
namespaceModule,
|
||||||
|
} from './orchest-identity';
|
||||||
|
|
||||||
|
describe('orchest-identity tenant routing', () => {
|
||||||
|
it('namespaceModule default and values', () => {
|
||||||
|
expect(namespaceModule(undefined)).toBe('intelli');
|
||||||
|
expect(namespaceModule('process')).toBe('process');
|
||||||
|
expect(namespaceModule('garbage')).toBe('intelli');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('tenantOrchestApiHost builds the namespace pattern', () => {
|
||||||
|
expect(tenantOrchestApiHost('acme', 'intelli')).toBe(
|
||||||
|
'orchest-api.orchest-intelli-acme.svc.cluster.local',
|
||||||
|
);
|
||||||
|
expect(tenantOrchestApiHost('acme', 'process')).toBe(
|
||||||
|
'orchest-api.orchest-process-acme.svc.cluster.local',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('isAllowedOrchestApiHost guards against malformed values', () => {
|
||||||
|
expect(
|
||||||
|
isAllowedOrchestApiHost('orchest-api.orchest-intelli-acme.svc.cluster.local'),
|
||||||
|
).toBe(true);
|
||||||
|
expect(isAllowedOrchestApiHost('evil.example.com')).toBe(false);
|
||||||
|
expect(
|
||||||
|
isAllowedOrchestApiHost('orchest-api.orchest-intelli-.svc.cluster.local'),
|
||||||
|
).toBe(false);
|
||||||
|
expect(
|
||||||
|
isAllowedOrchestApiHost('orchest-api.orchest-other-acme.svc.cluster.local'),
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run test to verify it fails**
|
||||||
|
|
||||||
|
Run: `npx jest src/modules/auth/orchest-identity.spec.ts -t 'tenant routing'`
|
||||||
|
Expected: FAIL — the three functions are not exported.
|
||||||
|
|
||||||
|
- [ ] **Step 3: Write minimal implementation (append to orchest-identity.ts)**
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
export function namespaceModule(raw: string | undefined): 'intelli' | 'process' {
|
||||||
|
return raw === 'process' ? 'process' : 'intelli';
|
||||||
|
}
|
||||||
|
|
||||||
|
export function tenantOrchestApiHost(
|
||||||
|
customerName: string,
|
||||||
|
module: string,
|
||||||
|
): string {
|
||||||
|
return `orchest-api.orchest-${module}-${customerName}.svc.cluster.local`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ORCHEST_API_HOST_RE =
|
||||||
|
/^orchest-api\.orchest-(intelli|process)-[a-z0-9-]+\.svc\.cluster\.local$/;
|
||||||
|
|
||||||
|
export function isAllowedOrchestApiHost(host: string): boolean {
|
||||||
|
return ORCHEST_API_HOST_RE.test(host);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Slug-normalization note (verify during implementation):** confirm the
|
||||||
|
JWT's `customer_name` is *exactly* the namespace slug (lowercase, kebab, no
|
||||||
|
spaces). If it is not, normalize deterministically inside
|
||||||
|
`tenantOrchestApiHost` (e.g. `customerName.toLowerCase().replace(/[^a-z0-9-]/g, '-')`)
|
||||||
|
and extend the test with the raw→normalized case. Do NOT guess the rule —
|
||||||
|
inspect a real token or ask the team.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run test to verify it passes**
|
||||||
|
|
||||||
|
Run: `npx jest src/modules/auth/orchest-identity.spec.ts -t 'tenant routing'`
|
||||||
|
Expected: PASS.
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/modules/auth/orchest-identity.ts src/modules/auth/orchest-identity.spec.ts
|
||||||
|
git commit -m "feat(orchest-identity): tenant orchest-api host derivation + allowlist"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 5: authz branch — call `/api/authz/check`, relay, fail-closed
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `src/modules/auth/auth.service.ts` (add `authorizeOrchestServiceAccess`)
|
||||||
|
- Modify: `src/modules/auth/auth.controller.ts` (insert the authz branch in `moduleIdentity`)
|
||||||
|
- Modify: `src/modules/auth/module-identity.controller.spec.ts`
|
||||||
|
|
||||||
|
**Interfaces:**
|
||||||
|
- Consumes: `tenantOrchestApiHost`, `isAllowedOrchestApiHost`, `namespaceModule` (Task 4); an HTTP client. Maestro uses gRPC for its own services but plain HTTP for this cross-service call — use `axios` if already a dependency, else the `@nestjs/axios` `HttpService`; confirm which is present before writing (grep `import axios` / `HttpService`).
|
||||||
|
- Produces: `AuthClientService.authorizeOrchestServiceAccess(args: { host: string; permission: string; projectUuid?: string; headers: Record<string,string> }): Promise<'allow' | 'deny' | 'error'>` — GET `http://{host}/api/authz/check?permission=…[&project_uuid=…]` with the identity headers; 200→`allow`, 403→`deny`, anything else/throw→`error`.
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the failing test (append to module-identity.controller.spec.ts)**
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
describe('GET /auth/module-identity — per-service authz', () => {
|
||||||
|
let controller: AuthController;
|
||||||
|
const auth = {
|
||||||
|
verifyAccessToken: jest.fn(),
|
||||||
|
authorizeOrchestServiceAccess: jest.fn(),
|
||||||
|
} as unknown as AuthClientService;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
jest.resetAllMocks();
|
||||||
|
process.env.ORCHEST_MODULE_PERMISSION_SEQID = '31';
|
||||||
|
process.env.ORCHEST_ADMIN_PERMISSION_SEQIDS = '34';
|
||||||
|
process.env.ORCHEST_NAMESPACE_MODULE = 'intelli';
|
||||||
|
const mod = await Test.createTestingModule({
|
||||||
|
controllers: [AuthController],
|
||||||
|
providers: [{ provide: AuthClientService, useValue: auth }],
|
||||||
|
}).compile(); // add the same stubbed providers as Task 3
|
||||||
|
controller = mod.get(AuthController);
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-1', username: 'alice', permissions: [31], customer_name: 'acme',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const q = { permission: 'session.open', project_uuid: 'p-1' };
|
||||||
|
|
||||||
|
it('has grant → 200 and calls the tenant host', async () => {
|
||||||
|
(auth.authorizeOrchestServiceAccess as jest.Mock).mockResolvedValue('allow');
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', q), r);
|
||||||
|
expect(r._status).toBe(200);
|
||||||
|
expect(auth.authorizeOrchestServiceAccess).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
host: 'orchest-api.orchest-intelli-acme.svc.cluster.local',
|
||||||
|
permission: 'session.open',
|
||||||
|
projectUuid: 'p-1',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('lacks grant → 403', async () => {
|
||||||
|
(auth.authorizeOrchestServiceAccess as jest.Mock).mockResolvedValue('deny');
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', q), r);
|
||||||
|
expect(r._status).toBe(403);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('orchest-api error → 502 (fail-closed)', async () => {
|
||||||
|
(auth.authorizeOrchestServiceAccess as jest.Mock).mockResolvedValue('error');
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', q), r);
|
||||||
|
expect(r._status).toBe(502);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('permission without project_uuid → 403 (all-or-nothing)', async () => {
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', { permission: 'session.open' }), r);
|
||||||
|
expect(r._status).toBe(403);
|
||||||
|
expect(auth.authorizeOrchestServiceAccess).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('no authz params → 200 identity-only (webserver case)', async () => {
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', {}), r);
|
||||||
|
expect(r._status).toBe(200);
|
||||||
|
expect(auth.authorizeOrchestServiceAccess).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('malformed customer_name → host fails allowlist → 403, no call', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-1', username: 'alice', permissions: [31], customer_name: 'Bad Name!',
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', q), r);
|
||||||
|
expect(r._status).toBe(403);
|
||||||
|
expect(auth.authorizeOrchestServiceAccess).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run test to verify it fails**
|
||||||
|
|
||||||
|
Run: `npx jest src/modules/auth/module-identity.controller.spec.ts -t 'per-service authz'`
|
||||||
|
Expected: FAIL — `authorizeOrchestServiceAccess` undefined / authz branch absent.
|
||||||
|
|
||||||
|
- [ ] **Step 3a: Implement the service helper**
|
||||||
|
|
||||||
|
In `auth.service.ts` (use the HTTP client confirmed in the Interfaces note; `axios` shown):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
public async authorizeOrchestServiceAccess(args: {
|
||||||
|
host: string;
|
||||||
|
permission: string;
|
||||||
|
projectUuid?: string;
|
||||||
|
headers: Record<string, string>;
|
||||||
|
}): Promise<'allow' | 'deny' | 'error'> {
|
||||||
|
const params: Record<string, string> = { permission: args.permission };
|
||||||
|
if (args.projectUuid) params.project_uuid = args.projectUuid;
|
||||||
|
try {
|
||||||
|
const resp = await axios.get(`http://${args.host}/api/authz/check`, {
|
||||||
|
params,
|
||||||
|
headers: args.headers,
|
||||||
|
timeout: 5000,
|
||||||
|
validateStatus: () => true, // never throw on 4xx/5xx; we branch below
|
||||||
|
});
|
||||||
|
if (resp.status === 200) return 'allow';
|
||||||
|
if (resp.status === 403) return 'deny';
|
||||||
|
return 'error';
|
||||||
|
} catch (e) {
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 3b: Insert the authz branch in the controller**
|
||||||
|
|
||||||
|
Replace the `// Phase 2 authz branch is inserted here` marker (Task 3) with:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const permission = req.query?.permission as string | undefined;
|
||||||
|
const projectUuid = req.query?.project_uuid as string | undefined;
|
||||||
|
if (permission) {
|
||||||
|
// Service-ingress caller: authorize per-project. All-or-nothing —
|
||||||
|
// an incomplete annotation must not silently skip the check.
|
||||||
|
if (!projectUuid) {
|
||||||
|
return res.status(403).send();
|
||||||
|
}
|
||||||
|
const module = namespaceModule(process.env.ORCHEST_NAMESPACE_MODULE);
|
||||||
|
const host = tenantOrchestApiHost(String(payload.customer_name ?? ''), module);
|
||||||
|
if (!isAllowedOrchestApiHost(host)) {
|
||||||
|
return res.status(403).send();
|
||||||
|
}
|
||||||
|
const identityHeaders: Record<string, string> = {
|
||||||
|
'X-Auth-User': String(payload.user_id),
|
||||||
|
'X-Auth-Username': String(payload.username ?? ''),
|
||||||
|
};
|
||||||
|
if (isAdmin(perms, parseAdminSeqids(process.env.ORCHEST_ADMIN_PERMISSION_SEQIDS))) {
|
||||||
|
identityHeaders['X-Auth-Roles'] = 'admin';
|
||||||
|
}
|
||||||
|
const decision = await this.authClient.authorizeOrchestServiceAccess({
|
||||||
|
host,
|
||||||
|
permission,
|
||||||
|
projectUuid,
|
||||||
|
headers: identityHeaders,
|
||||||
|
});
|
||||||
|
if (decision === 'deny') return res.status(403).send();
|
||||||
|
if (decision === 'error') return res.status(502).send();
|
||||||
|
// 'allow' falls through to the 200 below (identity headers already set).
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.status(200).send();
|
||||||
|
```
|
||||||
|
|
||||||
|
Add the imports `tenantOrchestApiHost`, `isAllowedOrchestApiHost`, `namespaceModule` to the existing `./orchest-identity` import line.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run test to verify it passes**
|
||||||
|
|
||||||
|
Run: `npx jest src/modules/auth/module-identity.controller.spec.ts`
|
||||||
|
Expected: PASS (identity + per-service authz suites).
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add src/modules/auth/auth.service.ts src/modules/auth/auth.controller.ts src/modules/auth/module-identity.controller.spec.ts
|
||||||
|
git commit -m "feat(auth): per-service authz branch on /auth/module-identity"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Task 6: orchest-side — scope the Maestro service auth-url (dbt-to-orchest repo)
|
||||||
|
|
||||||
|
**Files:**
|
||||||
|
- Modify: `lib/python/orchest-internals/_orchest/internals/utils.py:19-51` (`service_access_auth_url`)
|
||||||
|
- Test: the module's existing test (grep `service_access_auth_url` under `lib/python/…/tests`; if none, create `lib/python/orchest-internals/tests/test_service_access_auth_url.py`)
|
||||||
|
|
||||||
|
**This task is in the `dbt-to-orchest` repo, not Maestro.** Do it on a branch there (e.g. off the current RBAC branch). It has no dependency on Tasks 1-5 compiling, but the annotation it produces is what Task 5 consumes at runtime.
|
||||||
|
|
||||||
|
**Interfaces:**
|
||||||
|
- Consumes: nothing new.
|
||||||
|
- Produces: `service_access_auth_url(base_auth_url, permission, project_uuid, auth_custom)` — in Maestro mode (`auth_custom=True`), returns `f"{base_auth_url}?permission={permission}"` (plus `&project_uuid=…` when set) instead of returning `base_auth_url` unchanged.
|
||||||
|
|
||||||
|
- [ ] **Step 1: Write the failing test**
|
||||||
|
|
||||||
|
```python
|
||||||
|
from _orchest.internals.utils import service_access_auth_url
|
||||||
|
|
||||||
|
def test_maestro_mode_appends_scoped_params():
|
||||||
|
url = service_access_auth_url(
|
||||||
|
"http://maestro/auth/module-identity", "session.open", "p-1",
|
||||||
|
auth_custom=True,
|
||||||
|
)
|
||||||
|
assert url == (
|
||||||
|
"http://maestro/auth/module-identity?permission=session.open&project_uuid=p-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_maestro_mode_without_project_uuid():
|
||||||
|
url = service_access_auth_url(
|
||||||
|
"http://maestro/auth/module-identity", "project.view", None,
|
||||||
|
auth_custom=True,
|
||||||
|
)
|
||||||
|
assert url == "http://maestro/auth/module-identity?permission=project.view"
|
||||||
|
|
||||||
|
def test_local_mode_unchanged():
|
||||||
|
url = service_access_auth_url(
|
||||||
|
"http://auth-server/auth", "session.open", "p-1", auth_custom=False,
|
||||||
|
)
|
||||||
|
assert url == (
|
||||||
|
"http://auth-server/auth/service-access?permission=session.open&project_uuid=p-1"
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] **Step 2: Run test to verify it fails**
|
||||||
|
|
||||||
|
Run (in the pod or a venv with the lib on path):
|
||||||
|
`python -m pytest lib/python/orchest-internals/tests/test_service_access_auth_url.py -q`
|
||||||
|
Expected: FAIL on the two Maestro cases (current code returns the base URL unchanged).
|
||||||
|
|
||||||
|
- [ ] **Step 3: Write minimal implementation**
|
||||||
|
|
||||||
|
Replace the `if auth_custom:` short-circuit in `utils.py`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
if auth_custom:
|
||||||
|
# Maestro mode: no /auth/service-access sibling route — the unified
|
||||||
|
# /auth/module-identity route authorizes when the annotation carries
|
||||||
|
# the scope. Append the same permission/project_uuid params. (Maestro
|
||||||
|
# derives the tenant orchest-api host from the JWT, not from the URL.)
|
||||||
|
query = f"permission={permission}"
|
||||||
|
if project_uuid:
|
||||||
|
query += f"&project_uuid={project_uuid}"
|
||||||
|
return f"{base_auth_url}?{query}"
|
||||||
|
```
|
||||||
|
|
||||||
|
Update the docstring's "falls back to the plain base_auth_url" paragraph to describe the new scoped behavior.
|
||||||
|
|
||||||
|
- [ ] **Step 4: Run test to verify it passes**
|
||||||
|
|
||||||
|
Run: `python -m pytest lib/python/orchest-internals/tests/test_service_access_auth_url.py -q`
|
||||||
|
Expected: PASS (all three).
|
||||||
|
|
||||||
|
- [ ] **Step 5: Commit (dbt-to-orchest repo)**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git add lib/python/orchest-internals/_orchest/internals/utils.py lib/python/orchest-internals/tests/test_service_access_auth_url.py
|
||||||
|
git commit -m "feat(rbac): scope Maestro-mode service auth-url (close direct-URL bypass)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Deployment & manual verification (after Tasks 1-6)
|
||||||
|
|
||||||
|
Not code steps — run after merging, per the spec §8/§10.
|
||||||
|
|
||||||
|
- [ ] **Maestro env** on the Orchest-serving deployment: `ORCHEST_MODULE_PERMISSION_SEQID=31`, `ORCHEST_ADMIN_PERMISSION_SEQIDS=34`, `ORCHEST_NAMESPACE_MODULE=intelli`.
|
||||||
|
- [ ] **OrchestCluster spec** (webserver ingress): `AuthCustom: true`, `AuthUrl: http://maestro.<maestro-ns>.svc.cluster.local/auth/module-identity`, `AuthSignin: <platform login URL>`.
|
||||||
|
- [ ] **⚠️ celery-worker rebuild (Phase 2 / Task 6):** `service_access_auth_url` runs in the **celery-worker baked image** — rebuild + roll it, and verify the scoped annotation on a **freshly launched** service ingress (`kubectl get ingress …`), never an existing one. (This gotcha cost a debugging cycle last week; see `docs/source/development/building_images_minikube.md`.)
|
||||||
|
- [ ] **Verify path A (spec §10):** curl-drive orchest-api with the `X-Auth-*` headers Maestro would set and watch the RBAC ladder + `external_admin` resolve, before standing up real Maestro.
|
||||||
|
- [ ] **Forgery checks (spec §9.11, §9.12):** a non-admin's forged `X-Auth-Roles: admin` through the ingress must not reach orchest-api as admin; `?permission=…` appended to the app URL must not trigger an authz check.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Self-Review
|
||||||
|
|
||||||
|
**Spec coverage:**
|
||||||
|
- §4.1 cookie auth + JWKS reuse → Task 2 (expose verify) + Task 3 (read cookie).
|
||||||
|
- §4.2 ladder (401/403/200 + authz + all-or-nothing + no refresh) → Task 3 (401/403/200) + Task 5 (authz, 502, all-or-nothing).
|
||||||
|
- §4.3 three headers, no email → Task 3 (sets exactly the three; email never set).
|
||||||
|
- §5 module gate + admin set (config, list) → Task 1 + Task 3.
|
||||||
|
- §5.1 tenant routing (namespace pattern, allowlist, slug note, dadosferademo2) → Task 4 + Task 5.
|
||||||
|
- §6 forgery boundary → deployment verification checklist.
|
||||||
|
- §7 service_access_auth_url scoping → Task 6.
|
||||||
|
- §8 deployment config → deployment checklist.
|
||||||
|
- §9 tests 1-12 → Tasks 1/3 (1-5), Task 5 (6-10), deployment checklist (11-12).
|
||||||
|
- §10 minikube verification → deployment checklist.
|
||||||
|
- §11 phase split → Phase 1 (Tasks 1-3) / Phase 2 (Tasks 4-6).
|
||||||
|
|
||||||
|
**Placeholder scan:** none — every code step has real content; the only deliberately-open items are flagged verify-steps (HTTP client choice in Task 5; `customer_name` slug normalization in Task 4), each with an explicit instruction to inspect rather than guess.
|
||||||
|
|
||||||
|
**Type consistency:** `verifyAccessToken` (Task 2) consumed in Tasks 3/5; `moduleIdentity(req,res)` signature identical across Tasks 3/5; helper names (`parseAdminSeqids`, `moduleGateSeqid`, `isModuleAllowed`, `isAdmin`, `tenantOrchestApiHost`, `isAllowedOrchestApiHost`, `namespaceModule`) defined in Tasks 1/4 and used verbatim in Tasks 3/5; `authorizeOrchestServiceAccess` return union `'allow'|'deny'|'error'` consistent between service (Task 5 3a) and controller (Task 5 3b).
|
||||||
+2
-934
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,9 @@ const config: Config.InitialOptions = {
|
|||||||
roots: ['<rootDir>/src/', '<rootDir>/test/'],
|
roots: ['<rootDir>/src/', '<rootDir>/test/'],
|
||||||
testRegex: '.*\\.(test|spec)\\.[jt]s$',
|
testRegex: '.*\\.(test|spec)\\.[jt]s$',
|
||||||
transform: { '\\.[jt]s$': 'ts-jest' },
|
transform: { '\\.[jt]s$': 'ts-jest' },
|
||||||
|
// Dummy values for env vars read at module-import time (see the setup file),
|
||||||
|
// so specs importing those modules don't crash on load.
|
||||||
|
setupFiles: ['<rootDir>/test/jest.setup-env.ts'],
|
||||||
collectCoverageFrom: ['**/*.[jt]s'],
|
collectCoverageFrom: ['**/*.[jt]s'],
|
||||||
coverageDirectory: 'coverage',
|
coverageDirectory: 'coverage',
|
||||||
coveragePathIgnorePatterns: [
|
coveragePathIgnorePatterns: [
|
||||||
|
|||||||
Generated
+4
-5
@@ -16,7 +16,7 @@
|
|||||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||||
"@aws-sdk/signature-v4": "^3.370.0",
|
"@aws-sdk/signature-v4": "^3.370.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack-v2": "3.40.0-beta.21",
|
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
@@ -1735,10 +1735,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/@dadosfera/protospack-v2": {
|
"node_modules/@dadosfera/protospack-v2": {
|
||||||
"version": "3.40.0-beta.21",
|
"version": "3.40.0-beta.14",
|
||||||
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.21.tgz",
|
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.40.0-beta.14.tgz",
|
||||||
"integrity": "sha512-1mfFCXc8BFiYHcmFJSXt2KLqB3mQ1oPKQ/X39LCoB8buQHewvH1R7I7ZB67nrBxQ6XiTq6ojPT6PQRRgJtFgBg==",
|
"integrity": "sha512-pv3pxq0x1XcBgf3ajD6QOFRLOduh8iEozKFA3AKlIW4gid+gT4iL0GcU2M+O7h0QFeO4JIzRZe/nEMN82nqk7A==",
|
||||||
"license": "ISC",
|
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"rxjs": "^7.5.5"
|
"rxjs": "^7.5.5"
|
||||||
|
|||||||
+1
-1
@@ -34,7 +34,7 @@
|
|||||||
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
"@aws-sdk/lib-dynamodb": "^3.414.0",
|
||||||
"@aws-sdk/signature-v4": "^3.370.0",
|
"@aws-sdk/signature-v4": "^3.370.0",
|
||||||
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
|
||||||
"@dadosfera/protospack-v2": "3.40.0-beta.21",
|
"@dadosfera/protospack-v2": "^3.40.0-beta.14",
|
||||||
"@grpc/grpc-js": "^1.9.3",
|
"@grpc/grpc-js": "^1.9.3",
|
||||||
"@grpc/proto-loader": "^0.7.9",
|
"@grpc/proto-loader": "^0.7.9",
|
||||||
"@nestjs/cli": "^9.5.0",
|
"@nestjs/cli": "^9.5.0",
|
||||||
|
|||||||
@@ -1,76 +0,0 @@
|
|||||||
#!/usr/bin/env node
|
|
||||||
/*
|
|
||||||
* CI guard: fail if @dadosfera/protospack-v2 is consumed from a LOCAL ref
|
|
||||||
* (file:/link:/git/relative path/bare tarball) instead of the CodeArtifact
|
|
||||||
* registry.
|
|
||||||
*
|
|
||||||
* Only local consumption is blocked. Versions published to CodeArtifact —
|
|
||||||
* including alpha/beta/rc prereleases produced by the alpha/beta branches —
|
|
||||||
* are fine; those resolve to a registry URL in the lockfile. The thing that
|
|
||||||
* must NOT reach beta (staging) or main (prod) is a dependency wired to a
|
|
||||||
* local `npm pack` tarball / overlay. Runs in the PR test workflow for PRs
|
|
||||||
* targeting beta/main and exits non-zero on any local ref.
|
|
||||||
*/
|
|
||||||
const fs = require('fs');
|
|
||||||
const path = require('path');
|
|
||||||
|
|
||||||
const PKG = '@dadosfera/protospack-v2';
|
|
||||||
const root = path.resolve(__dirname, '..');
|
|
||||||
const pkg = JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf8'));
|
|
||||||
|
|
||||||
const problems = [];
|
|
||||||
|
|
||||||
// A dependency SPEC is local if it's a filesystem path, symlink, git ref, or a
|
|
||||||
// bare tarball path. A plain semver (incl. prereleases like 3.35.0-beta.1)
|
|
||||||
// resolves from the registry and is allowed.
|
|
||||||
function isLocalSpec(spec) {
|
|
||||||
return /^(file:|link:|git[:+]|\.\.?\/|\/|~\/)/.test(spec) || spec.endsWith('.tgz');
|
|
||||||
}
|
|
||||||
|
|
||||||
const spec =
|
|
||||||
(pkg.dependencies && pkg.dependencies[PKG]) ||
|
|
||||||
(pkg.devDependencies && pkg.devDependencies[PKG]);
|
|
||||||
|
|
||||||
if (!spec) {
|
|
||||||
problems.push(`${PKG} is not listed as a dependency at all.`);
|
|
||||||
} else if (isLocalSpec(spec)) {
|
|
||||||
problems.push(`${PKG} points at a local path/tarball/git ref: "${spec}".`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Also catch a lockfile resolved to a LOCAL ref even if package.json looks
|
|
||||||
// clean. A registry URL (https://.../-/*.tgz) is the normal published
|
|
||||||
// resolution and is fine — only file: refs and bare local tarball paths
|
|
||||||
// (no http host) are blocked. Prerelease VERSIONS are not flagged: an
|
|
||||||
// alpha/beta/rc published to CodeArtifact resolves to a registry URL.
|
|
||||||
const lockPath = path.join(root, 'package-lock.json');
|
|
||||||
if (fs.existsSync(lockPath)) {
|
|
||||||
const lock = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
|
|
||||||
const nodes = { ...(lock.packages || {}), ...(lock.dependencies || {}) };
|
|
||||||
for (const [name, node] of Object.entries(nodes)) {
|
|
||||||
if (!name.includes('protospack-v2') || !node) continue;
|
|
||||||
const resolved = node.resolved || '';
|
|
||||||
const isLocal =
|
|
||||||
resolved.startsWith('file:') ||
|
|
||||||
(resolved.endsWith('.tgz') && !/^https?:\/\//.test(resolved));
|
|
||||||
if (isLocal) {
|
|
||||||
problems.push(
|
|
||||||
`package-lock.json resolves ${PKG} to a local ref: "${resolved}".`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (problems.length) {
|
|
||||||
console.error('✗ protospack-v2 dependency guard FAILED:');
|
|
||||||
for (const p of problems) console.error(' - ' + p);
|
|
||||||
console.error(
|
|
||||||
'\nMerging to beta/main requires ' +
|
|
||||||
PKG +
|
|
||||||
' to come from CodeArtifact, not a local tarball/overlay. Publish ' +
|
|
||||||
'protospack-v2 (a beta prerelease is fine for the beta branch) and ' +
|
|
||||||
'repoint this dependency before merging.',
|
|
||||||
);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`✓ ${PKG} is consumed from the registry: "${spec}"`);
|
|
||||||
@@ -1,122 +0,0 @@
|
|||||||
import { BadRequestException } from '@nestjs/common';
|
|
||||||
import { PipelineExecutionGuard } from './pipeline-execution.guard';
|
|
||||||
|
|
||||||
const logger = { info: jest.fn(), error: jest.fn() };
|
|
||||||
|
|
||||||
type Routes = {
|
|
||||||
pipeline?: unknown | Error;
|
|
||||||
runs?: unknown | Error;
|
|
||||||
};
|
|
||||||
|
|
||||||
// The guard makes up to two platform-api reads: GET /pipeline/{id} (type)
|
|
||||||
// and, for batch only, GET /pipeline/{id}/pipeline_run (last run).
|
|
||||||
function buildGuard(routes: Routes) {
|
|
||||||
const proxy = jest.fn((method: string, path: string) => {
|
|
||||||
const answer = path.endsWith('/pipeline_run') ? routes.runs : routes.pipeline;
|
|
||||||
return answer instanceof Error ? Promise.reject(answer) : Promise.resolve(answer);
|
|
||||||
});
|
|
||||||
const guard = new PipelineExecutionGuard(
|
|
||||||
{ logger } as unknown as ConstructorParameters<typeof PipelineExecutionGuard>[0],
|
|
||||||
{ proxy } as unknown as ConstructorParameters<typeof PipelineExecutionGuard>[1],
|
|
||||||
);
|
|
||||||
return { guard, proxy };
|
|
||||||
}
|
|
||||||
|
|
||||||
function contextWith(pipelineId = 'abc-123') {
|
|
||||||
return {
|
|
||||||
switchToHttp: () => ({
|
|
||||||
getRequest: () => ({ params: { pipelineId }, user: {} }),
|
|
||||||
}),
|
|
||||||
} as unknown as Parameters<PipelineExecutionGuard['canActivate']>[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
const cdcPipeline = { jobs: [{ job_id: 'p_0', input: { connector: 'cdc', table_name: 't' } }] };
|
|
||||||
const batchPipeline = { jobs: [{ job_id: 'p_0', input: { connector: 'jdbc', table_name: 't' } }] };
|
|
||||||
|
|
||||||
describe('PipelineExecutionGuard', () => {
|
|
||||||
afterEach(() => jest.clearAllMocks());
|
|
||||||
|
|
||||||
describe('CDC pipelines (explicit connector type)', () => {
|
|
||||||
it('allows the edit without consulting run history', async () => {
|
|
||||||
const { guard, proxy } = buildGuard({ pipeline: cdcPipeline });
|
|
||||||
await expect(guard.canActivate(contextWith())).resolves.toBe(true);
|
|
||||||
expect(proxy).toHaveBeenCalledTimes(1);
|
|
||||||
expect(proxy).toHaveBeenCalledWith('GET', '/pipeline/abc_123', {});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('is CDC when any job is a CDC job', async () => {
|
|
||||||
const mixed = { jobs: [...batchPipeline.jobs, ...cdcPipeline.jobs] };
|
|
||||||
const { guard, proxy } = buildGuard({ pipeline: mixed });
|
|
||||||
await expect(guard.canActivate(contextWith())).resolves.toBe(true);
|
|
||||||
expect(proxy).toHaveBeenCalledTimes(1);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('batch pipelines', () => {
|
|
||||||
it('allows the edit when the pipeline never ran (empty run history)', async () => {
|
|
||||||
const { guard } = buildGuard({ pipeline: batchPipeline, runs: [] });
|
|
||||||
await expect(guard.canActivate(contextWith())).resolves.toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('allows the edit when the last run has no last_status', async () => {
|
|
||||||
const { guard } = buildGuard({ pipeline: batchPipeline, runs: [{}] });
|
|
||||||
await expect(guard.canActivate(contextWith())).resolves.toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('allows the edit when the pipeline is not running', async () => {
|
|
||||||
const { guard } = buildGuard({
|
|
||||||
pipeline: batchPipeline,
|
|
||||||
runs: [{ last_status: 'SUCCEEDED' }],
|
|
||||||
});
|
|
||||||
await expect(guard.canActivate(contextWith())).resolves.toBe(true);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('blocks with the is-running message when the pipeline is running', async () => {
|
|
||||||
const { guard } = buildGuard({
|
|
||||||
pipeline: batchPipeline,
|
|
||||||
runs: [{ last_status: 'RUNNING' }],
|
|
||||||
});
|
|
||||||
await expect(guard.canActivate(contextWith())).rejects.toThrow(
|
|
||||||
'Pipeline is running, cannot update input now',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('treats a pipeline with no jobs as batch and checks its runs', async () => {
|
|
||||||
const { guard, proxy } = buildGuard({ pipeline: { jobs: [] }, runs: [] });
|
|
||||||
await expect(guard.canActivate(contextWith())).resolves.toBe(true);
|
|
||||||
expect(proxy).toHaveBeenCalledTimes(2);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('does not double-wrap the is-running BadRequestException', async () => {
|
|
||||||
const { guard } = buildGuard({
|
|
||||||
pipeline: batchPipeline,
|
|
||||||
runs: [{ last_status: 'running' }],
|
|
||||||
});
|
|
||||||
await expect(guard.canActivate(contextWith())).rejects.toBeInstanceOf(
|
|
||||||
BadRequestException,
|
|
||||||
);
|
|
||||||
await expect(guard.canActivate(contextWith())).rejects.not.toThrow(
|
|
||||||
/Error checking pipeline status/,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('failures', () => {
|
|
||||||
it('fails closed when the pipeline read fails', async () => {
|
|
||||||
const { guard } = buildGuard({ pipeline: new Error('platform down') });
|
|
||||||
await expect(guard.canActivate(contextWith())).rejects.toThrow(
|
|
||||||
'Error checking pipeline status: platform down',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('fails closed when the run-history read fails', async () => {
|
|
||||||
const { guard } = buildGuard({
|
|
||||||
pipeline: batchPipeline,
|
|
||||||
runs: new Error('runs down'),
|
|
||||||
});
|
|
||||||
await expect(guard.canActivate(contextWith())).rejects.toThrow(
|
|
||||||
'Error checking pipeline status: runs down',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -16,7 +16,6 @@ import {
|
|||||||
import { PipelinesClientConfiguration } from 'src/modules/pipelinesV2/pipelines-client';
|
import { PipelinesClientConfiguration } from 'src/modules/pipelinesV2/pipelines-client';
|
||||||
import { PlatformApiService } from 'src/modules/platform-api/platform-api.service';
|
import { PlatformApiService } from 'src/modules/platform-api/platform-api.service';
|
||||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
||||||
import { isCdcPipeline } from 'src/utils/cdc';
|
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class PipelineExecutionGuard implements CanActivate {
|
export class PipelineExecutionGuard implements CanActivate {
|
||||||
@@ -40,51 +39,27 @@ export class PipelineExecutionGuard implements CanActivate {
|
|||||||
const user = request.user;
|
const user = request.user;
|
||||||
const idRegex = /[^0-9a-zA-Z_$]+/g;
|
const idRegex = /[^0-9a-zA-Z_$]+/g;
|
||||||
const convertedId = pipelineId.replace(idRegex, '_');
|
const convertedId = pipelineId.replace(idRegex, '_');
|
||||||
|
|
||||||
// Decide by the pipeline's explicit type (platform job
|
|
||||||
// `input.connector`), never by the absence of run history.
|
|
||||||
const pipeline = await this.platformApiService.proxy(
|
|
||||||
'GET',
|
|
||||||
`/pipeline/${convertedId}`,
|
|
||||||
user,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (isCdcPipeline(pipeline)) {
|
|
||||||
// CDC pipelines have no batch runs; the platform-api gates connector
|
|
||||||
// edits itself (require_pipeline_running on add/remove tables).
|
|
||||||
this.logger.info('PipelineExecutionGuard: CDC pipeline, no run to block on');
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
const status = await this.platformApiService.proxy(
|
const status = await this.platformApiService.proxy(
|
||||||
'GET',
|
'GET',
|
||||||
`/pipeline/${convertedId}/pipeline_run`,
|
`/pipeline/${convertedId}/pipeline_run`,
|
||||||
user,
|
user,
|
||||||
);
|
);
|
||||||
|
|
||||||
const currentStatus = status?.[status.length - 1];
|
const currentStatus = status[status.length - 1]
|
||||||
|
|
||||||
this.logger.info('Pipeline current status response:' + JSON.stringify(currentStatus));
|
this.logger.info('Pipeline current status response:' + JSON.stringify(currentStatus));
|
||||||
|
|
||||||
// Batch pipeline that never ran yet: nothing can be executing.
|
|
||||||
if (!currentStatus?.last_status) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (currentStatus.last_status.toLowerCase() === 'running') {
|
if (currentStatus.last_status.toLowerCase() === 'running') {
|
||||||
this.logger.error('Pipeline is running, cannot update input now');
|
this.logger.error('Pipeline is running, cannot update input now');
|
||||||
throw new BadRequestException('Pipeline is running, cannot update input now');
|
throw new BadRequestException('Pipeline is running, cannot update input now');
|
||||||
|
} else {
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
// Preserve the deliberate is-running rejection; only wrap genuine
|
|
||||||
// status-check failures (fail closed on those for a destructive gate).
|
|
||||||
if (error instanceof BadRequestException) {
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
this.logger.error('Error in PipelineExecutionGuard: ' + error.message);
|
this.logger.error('Error in PipelineExecutionGuard: ' + error.message);
|
||||||
throw new BadRequestException('Error checking pipeline status: ' + error.message);
|
throw new BadRequestException('Error checking pipeline status: ' + error.message);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,12 +32,20 @@ import {
|
|||||||
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
|
||||||
|
|
||||||
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
|
||||||
|
import {
|
||||||
|
parseAdminSeqids,
|
||||||
|
moduleGateSeqid,
|
||||||
|
isModuleAllowed,
|
||||||
|
isAdmin,
|
||||||
|
tenantOrchestApiHost,
|
||||||
|
isAllowedOrchestApiHost,
|
||||||
|
namespaceModule,
|
||||||
|
} from './orchest-identity';
|
||||||
import {
|
import {
|
||||||
Authenticated,
|
Authenticated,
|
||||||
RequireAllPermissions,
|
RequireAllPermissions,
|
||||||
} from 'src/decorators/authentication.decorator';
|
} from 'src/decorators/authentication.decorator';
|
||||||
import { AuthClientService } from './auth.service';
|
import { AuthClientService } from './auth.service';
|
||||||
import { UserDTO } from './dtos/login';
|
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
|
import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
|
||||||
import { RequestUser, User } from 'src/decorators/user.decorator';
|
import { RequestUser, User } from 'src/decorators/user.decorator';
|
||||||
@@ -487,7 +495,7 @@ export class AuthController {
|
|||||||
this.logger.info('Authenticating via X-Api-key header');
|
this.logger.info('Authenticating via X-Api-key header');
|
||||||
const { api_key } = await this.apiKeyService.get(apiKey);
|
const { api_key } = await this.apiKeyService.get(apiKey);
|
||||||
|
|
||||||
const userDto: UserDTO = {
|
const userDto = {
|
||||||
id: api_key.user_id,
|
id: api_key.user_id,
|
||||||
name: api_key.username,
|
name: api_key.username,
|
||||||
email: api_key.username,
|
email: api_key.username,
|
||||||
@@ -495,8 +503,7 @@ export class AuthController {
|
|||||||
id: api_key.customer_id,
|
id: api_key.customer_id,
|
||||||
name: api_key.customer_name,
|
name: api_key.customer_name,
|
||||||
tier: api_key.customer_tier,
|
tier: api_key.customer_tier,
|
||||||
},
|
}
|
||||||
permissions: [],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return res.status(200).json(userDto);
|
return res.status(200).json(userDto);
|
||||||
@@ -533,4 +540,84 @@ export class AuthController {
|
|||||||
return res.status(200).json(user);
|
return res.status(200).json(user);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ingress auth subrequest for Orchest (see dbt-to-orchest
|
||||||
|
* docs/superpowers/specs/2026-08-20-maestro-module-identity-design.md).
|
||||||
|
*
|
||||||
|
* Authenticates the ddf-auth cookie, gates on the module permission, and
|
||||||
|
* returns the X-Auth-* identity headers orchest-api trusts. Never writes
|
||||||
|
* cookies (an auth_request response cannot). No token refresh: an expired
|
||||||
|
* token is a 401 (the signin flow re-auths).
|
||||||
|
*/
|
||||||
|
@Get('module-identity')
|
||||||
|
async moduleIdentity(@Req() req: Request, @Res() res: Response) {
|
||||||
|
const token = req.cookies?.['ddf-auth'];
|
||||||
|
if (!token) {
|
||||||
|
return res.status(401).send();
|
||||||
|
}
|
||||||
|
|
||||||
|
let payload: any;
|
||||||
|
try {
|
||||||
|
payload = await this.authClient.verifyAccessToken(token);
|
||||||
|
} catch (e) {
|
||||||
|
return res.status(401).send();
|
||||||
|
}
|
||||||
|
|
||||||
|
const perms: number[] = payload?.permissions ?? [];
|
||||||
|
const gate = moduleGateSeqid(process.env.ORCHEST_MODULE_PERMISSION_SEQID);
|
||||||
|
if (!isModuleAllowed(perms, gate)) {
|
||||||
|
return res.status(403).send();
|
||||||
|
}
|
||||||
|
|
||||||
|
const adminUser = isAdmin(
|
||||||
|
perms,
|
||||||
|
parseAdminSeqids(process.env.ORCHEST_ADMIN_PERMISSION_SEQIDS),
|
||||||
|
);
|
||||||
|
|
||||||
|
res.set('X-Auth-User', String(payload.user_id));
|
||||||
|
res.set('X-Auth-Username', String(payload.username ?? ''));
|
||||||
|
if (adminUser) {
|
||||||
|
res.set('X-Auth-Roles', 'admin');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Service-ingress caller: the orchest-api-authored annotation carries the
|
||||||
|
// scope. Authorize per-project against the tenant's orchest-api. A
|
||||||
|
// param-less request is the webserver-ingress case → identity only.
|
||||||
|
const permission = req.query?.permission as string | undefined;
|
||||||
|
const projectUuid = req.query?.project_uuid as string | undefined;
|
||||||
|
if (permission) {
|
||||||
|
// All-or-nothing: an incomplete annotation must not silently skip the
|
||||||
|
// per-project check.
|
||||||
|
if (!projectUuid) {
|
||||||
|
return res.status(403).send();
|
||||||
|
}
|
||||||
|
const module = namespaceModule(process.env.ORCHEST_NAMESPACE_MODULE);
|
||||||
|
const host = tenantOrchestApiHost(
|
||||||
|
String(payload.customer_name ?? ''),
|
||||||
|
module,
|
||||||
|
);
|
||||||
|
if (!isAllowedOrchestApiHost(host)) {
|
||||||
|
return res.status(403).send();
|
||||||
|
}
|
||||||
|
const identityHeaders: Record<string, string> = {
|
||||||
|
'X-Auth-User': String(payload.user_id),
|
||||||
|
'X-Auth-Username': String(payload.username ?? ''),
|
||||||
|
};
|
||||||
|
if (adminUser) {
|
||||||
|
identityHeaders['X-Auth-Roles'] = 'admin';
|
||||||
|
}
|
||||||
|
const decision = await this.authClient.authorizeOrchestServiceAccess({
|
||||||
|
host,
|
||||||
|
permission,
|
||||||
|
projectUuid,
|
||||||
|
headers: identityHeaders,
|
||||||
|
});
|
||||||
|
if (decision === 'deny') return res.status(403).send();
|
||||||
|
if (decision === 'error') return res.status(502).send();
|
||||||
|
// 'allow' falls through to the 200 (identity headers already set).
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.status(200).send();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
import { ClientGrpc } from '@nestjs/microservices';
|
import { ClientGrpc } from '@nestjs/microservices';
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
import { lastValueFrom } from 'rxjs';
|
import { lastValueFrom } from 'rxjs';
|
||||||
|
import axios from 'axios';
|
||||||
|
|
||||||
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
|
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
|
||||||
import {
|
import {
|
||||||
@@ -307,7 +308,7 @@ export class AuthClientService implements OnModuleInit {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public async validateUserSession(accessToken: any, resourceHost: string) {
|
public async validateUserSession(accessToken: any, resourceHost: string) {
|
||||||
const payload = await this.validateJwtToken(accessToken);
|
const payload = await this.verifyAccessToken(accessToken);
|
||||||
|
|
||||||
const userDto = await this.getUserfromPayload(payload);
|
const userDto = await this.getUserfromPayload(payload);
|
||||||
|
|
||||||
@@ -410,7 +411,12 @@ export class AuthClientService implements OnModuleInit {
|
|||||||
this.logger.info('Clean cookie sessions');
|
this.logger.info('Clean cookie sessions');
|
||||||
}
|
}
|
||||||
|
|
||||||
private async validateJwtToken(token: string) {
|
/**
|
||||||
|
* Verify a DUC access token against the JWKS and return its payload.
|
||||||
|
* Public so the /auth/module-identity route can authenticate the ddf-auth
|
||||||
|
* cookie the same way (was the private validateJwtToken).
|
||||||
|
*/
|
||||||
|
public async verifyAccessToken(token: string) {
|
||||||
const decoded: any = token && jwt.decode(token, { complete: true });
|
const decoded: any = token && jwt.decode(token, { complete: true });
|
||||||
if (!decoded) throw new Error('Invalid token');
|
if (!decoded) throw new Error('Invalid token');
|
||||||
|
|
||||||
@@ -447,9 +453,6 @@ export class AuthClientService implements OnModuleInit {
|
|||||||
name: payload.customer_name,
|
name: payload.customer_name,
|
||||||
tier: payload.customer_tier,
|
tier: payload.customer_tier,
|
||||||
},
|
},
|
||||||
// Raw permission seqids from the JWT. Consumers own the seqid->meaning
|
|
||||||
// mapping (e.g. Orchest's auth-server); Maestro reports them as-is.
|
|
||||||
permissions: payload.permissions ?? [],
|
|
||||||
};
|
};
|
||||||
|
|
||||||
return userDto;
|
return userDto;
|
||||||
@@ -495,4 +498,33 @@ export class AuthClientService implements OnModuleInit {
|
|||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ask a tenant's orchest-api whether the acting user holds `permission` on
|
||||||
|
* `projectUuid` (the per-service authz half of /auth/module-identity).
|
||||||
|
* Fail-closed: any error / unexpected status → 'error' (the route denies).
|
||||||
|
*/
|
||||||
|
public async authorizeOrchestServiceAccess(args: {
|
||||||
|
host: string;
|
||||||
|
permission: string;
|
||||||
|
projectUuid?: string;
|
||||||
|
headers: Record<string, string>;
|
||||||
|
}): Promise<'allow' | 'deny' | 'error'> {
|
||||||
|
const params: Record<string, string> = { permission: args.permission };
|
||||||
|
if (args.projectUuid) params.project_uuid = args.projectUuid;
|
||||||
|
try {
|
||||||
|
const resp = await axios.get(`http://${args.host}/api/authz/check`, {
|
||||||
|
params,
|
||||||
|
headers: args.headers,
|
||||||
|
timeout: 5000,
|
||||||
|
// Never throw on 4xx/5xx; branch on the status ourselves.
|
||||||
|
validateStatus: () => true,
|
||||||
|
});
|
||||||
|
if (resp.status === 200) return 'allow';
|
||||||
|
if (resp.status === 403) return 'deny';
|
||||||
|
return 'error';
|
||||||
|
} catch (e) {
|
||||||
|
return 'error';
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -152,6 +152,5 @@ export type UserDTO = {
|
|||||||
id: string,
|
id: string,
|
||||||
name: string,
|
name: string,
|
||||||
tier: string,
|
tier: string,
|
||||||
},
|
}
|
||||||
permissions: number[],
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,206 @@
|
|||||||
|
import { Test } from '@nestjs/testing';
|
||||||
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
import { AuthController } from './auth.controller';
|
||||||
|
import { AuthClientService } from './auth.service';
|
||||||
|
import { ApiKeyService } from 'src/modules/api-key/api-key.service';
|
||||||
|
|
||||||
|
function res() {
|
||||||
|
const headers: Record<string, string> = {};
|
||||||
|
const r: any = {
|
||||||
|
_status: 0,
|
||||||
|
_sent: undefined,
|
||||||
|
set: (k: string, v: string) => {
|
||||||
|
headers[k] = v;
|
||||||
|
return r;
|
||||||
|
},
|
||||||
|
status: (c: number) => {
|
||||||
|
r._status = c;
|
||||||
|
return r;
|
||||||
|
},
|
||||||
|
send: (b?: any) => {
|
||||||
|
r._sent = b ?? '';
|
||||||
|
return r;
|
||||||
|
},
|
||||||
|
json: (b?: any) => {
|
||||||
|
r._sent = b;
|
||||||
|
return r;
|
||||||
|
},
|
||||||
|
_headers: headers,
|
||||||
|
};
|
||||||
|
return r;
|
||||||
|
}
|
||||||
|
|
||||||
|
function req(cookie?: string, query: Record<string, string> = {}) {
|
||||||
|
return { cookies: cookie ? { 'ddf-auth': cookie } : {}, query } as any;
|
||||||
|
}
|
||||||
|
|
||||||
|
const loggerStub = {
|
||||||
|
logger: { info: jest.fn(), error: jest.fn(), warn: jest.fn() },
|
||||||
|
} as unknown as DadosferaLogger;
|
||||||
|
|
||||||
|
async function makeController(auth: AuthClientService): Promise<AuthController> {
|
||||||
|
const mod = await Test.createTestingModule({
|
||||||
|
controllers: [AuthController],
|
||||||
|
providers: [
|
||||||
|
{ provide: DadosferaLogger, useValue: loggerStub },
|
||||||
|
{ provide: AuthClientService, useValue: auth },
|
||||||
|
{ provide: ApiKeyService, useValue: {} },
|
||||||
|
],
|
||||||
|
}).compile();
|
||||||
|
return mod.get(AuthController);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('GET /auth/module-identity — identity', () => {
|
||||||
|
let controller: AuthController;
|
||||||
|
const auth = {
|
||||||
|
verifyAccessToken: jest.fn(),
|
||||||
|
} as unknown as AuthClientService;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
jest.resetAllMocks();
|
||||||
|
process.env.ORCHEST_MODULE_PERMISSION_SEQID = '31';
|
||||||
|
process.env.ORCHEST_ADMIN_PERMISSION_SEQIDS = '34';
|
||||||
|
controller = await makeController(auth);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('no cookie → 401', async () => {
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req(undefined), r);
|
||||||
|
expect(r._status).toBe(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('valid + module + admin → 200 with X-Auth-Roles: admin', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-1',
|
||||||
|
username: 'alice',
|
||||||
|
permissions: [31, 34],
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._status).toBe(200);
|
||||||
|
expect(r._headers['X-Auth-User']).toBe('u-1');
|
||||||
|
expect(r._headers['X-Auth-Username']).toBe('alice');
|
||||||
|
expect(r._headers['X-Auth-Roles']).toBe('admin');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('valid + module, not admin → 200, no X-Auth-Roles', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-2',
|
||||||
|
username: 'bob',
|
||||||
|
permissions: [31],
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._status).toBe(200);
|
||||||
|
expect(r._headers['X-Auth-Roles']).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('valid, lacks module → 403', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-3',
|
||||||
|
username: 'carol',
|
||||||
|
permissions: [5],
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._status).toBe(403);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('verify throws (expired/bad) → 401', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockRejectedValue(new Error('bad'));
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._status).toBe(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('admin seqids extended by config → 200 admin', async () => {
|
||||||
|
process.env.ORCHEST_ADMIN_PERMISSION_SEQIDS = '34,99';
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-4',
|
||||||
|
username: 'dana',
|
||||||
|
permissions: [31, 99],
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok'), r);
|
||||||
|
expect(r._headers['X-Auth-Roles']).toBe('admin');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('GET /auth/module-identity — per-service authz', () => {
|
||||||
|
let controller: AuthController;
|
||||||
|
const auth = {
|
||||||
|
verifyAccessToken: jest.fn(),
|
||||||
|
authorizeOrchestServiceAccess: jest.fn(),
|
||||||
|
} as unknown as AuthClientService;
|
||||||
|
|
||||||
|
const q = { permission: 'session.open', project_uuid: 'p-1' };
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
jest.resetAllMocks();
|
||||||
|
process.env.ORCHEST_MODULE_PERMISSION_SEQID = '31';
|
||||||
|
process.env.ORCHEST_ADMIN_PERMISSION_SEQIDS = '34';
|
||||||
|
process.env.ORCHEST_NAMESPACE_MODULE = 'intelli';
|
||||||
|
controller = await makeController(auth);
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-1',
|
||||||
|
username: 'alice',
|
||||||
|
permissions: [31],
|
||||||
|
customer_name: 'acme',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('has grant → 200 and calls the tenant host', async () => {
|
||||||
|
(auth.authorizeOrchestServiceAccess as jest.Mock).mockResolvedValue('allow');
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', q), r);
|
||||||
|
expect(r._status).toBe(200);
|
||||||
|
expect(auth.authorizeOrchestServiceAccess).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
host: 'orchest-api.orchest-intelli-acme.svc.cluster.local',
|
||||||
|
permission: 'session.open',
|
||||||
|
projectUuid: 'p-1',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('lacks grant → 403', async () => {
|
||||||
|
(auth.authorizeOrchestServiceAccess as jest.Mock).mockResolvedValue('deny');
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', q), r);
|
||||||
|
expect(r._status).toBe(403);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('orchest-api error → 502 (fail-closed)', async () => {
|
||||||
|
(auth.authorizeOrchestServiceAccess as jest.Mock).mockResolvedValue('error');
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', q), r);
|
||||||
|
expect(r._status).toBe(502);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('permission without project_uuid → 403 (all-or-nothing)', async () => {
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', { permission: 'session.open' }), r);
|
||||||
|
expect(r._status).toBe(403);
|
||||||
|
expect(auth.authorizeOrchestServiceAccess).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('no authz params → 200 identity-only (webserver case)', async () => {
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', {}), r);
|
||||||
|
expect(r._status).toBe(200);
|
||||||
|
expect(auth.authorizeOrchestServiceAccess).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('malformed customer_name → host fails allowlist → 403, no call', async () => {
|
||||||
|
(auth.verifyAccessToken as jest.Mock).mockResolvedValue({
|
||||||
|
user_id: 'u-1',
|
||||||
|
username: 'alice',
|
||||||
|
permissions: [31],
|
||||||
|
customer_name: '',
|
||||||
|
});
|
||||||
|
const r = res();
|
||||||
|
await controller.moduleIdentity(req('tok', q), r);
|
||||||
|
expect(r._status).toBe(403);
|
||||||
|
expect(auth.authorizeOrchestServiceAccess).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
import {
|
||||||
|
parseAdminSeqids,
|
||||||
|
moduleGateSeqid,
|
||||||
|
isModuleAllowed,
|
||||||
|
isAdmin,
|
||||||
|
tenantOrchestApiHost,
|
||||||
|
isAllowedOrchestApiHost,
|
||||||
|
namespaceModule,
|
||||||
|
} from './orchest-identity';
|
||||||
|
|
||||||
|
describe('orchest-identity mapping', () => {
|
||||||
|
it('parseAdminSeqids: default, single, list, whitespace', () => {
|
||||||
|
expect(parseAdminSeqids(undefined)).toEqual([34]);
|
||||||
|
expect(parseAdminSeqids('')).toEqual([34]);
|
||||||
|
expect(parseAdminSeqids('34')).toEqual([34]);
|
||||||
|
expect(parseAdminSeqids('34,40')).toEqual([34, 40]);
|
||||||
|
expect(parseAdminSeqids(' 34 , 40 ')).toEqual([34, 40]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('moduleGateSeqid: default and override', () => {
|
||||||
|
expect(moduleGateSeqid(undefined)).toBe(31);
|
||||||
|
expect(moduleGateSeqid('43')).toBe(43);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('isModuleAllowed', () => {
|
||||||
|
expect(isModuleAllowed([31, 5], 31)).toBe(true);
|
||||||
|
expect(isModuleAllowed([5, 7], 31)).toBe(false);
|
||||||
|
expect(isModuleAllowed([], 31)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('isAdmin: intersection', () => {
|
||||||
|
expect(isAdmin([31, 34], [34])).toBe(true);
|
||||||
|
expect(isAdmin([31], [34])).toBe(false);
|
||||||
|
expect(isAdmin([99], [34, 99])).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('orchest-identity tenant routing', () => {
|
||||||
|
it('namespaceModule default and values', () => {
|
||||||
|
expect(namespaceModule(undefined)).toBe('intelli');
|
||||||
|
expect(namespaceModule('process')).toBe('process');
|
||||||
|
expect(namespaceModule('garbage')).toBe('intelli');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('tenantOrchestApiHost builds the namespace pattern', () => {
|
||||||
|
expect(tenantOrchestApiHost('acme', 'intelli')).toBe(
|
||||||
|
'orchest-api.orchest-intelli-acme.svc.cluster.local',
|
||||||
|
);
|
||||||
|
expect(tenantOrchestApiHost('acme', 'process')).toBe(
|
||||||
|
'orchest-api.orchest-process-acme.svc.cluster.local',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('tenantOrchestApiHost normalizes a non-slug customer_name', () => {
|
||||||
|
expect(tenantOrchestApiHost('Acme Corp', 'intelli')).toBe(
|
||||||
|
'orchest-api.orchest-intelli-acme-corp.svc.cluster.local',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('isAllowedOrchestApiHost guards against malformed values', () => {
|
||||||
|
expect(
|
||||||
|
isAllowedOrchestApiHost(
|
||||||
|
'orchest-api.orchest-intelli-acme.svc.cluster.local',
|
||||||
|
),
|
||||||
|
).toBe(true);
|
||||||
|
expect(isAllowedOrchestApiHost('evil.example.com')).toBe(false);
|
||||||
|
expect(
|
||||||
|
isAllowedOrchestApiHost('orchest-api.orchest-intelli-.svc.cluster.local'),
|
||||||
|
).toBe(false);
|
||||||
|
expect(
|
||||||
|
isAllowedOrchestApiHost(
|
||||||
|
'orchest-api.orchest-other-acme.svc.cluster.local',
|
||||||
|
),
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
/**
|
||||||
|
* Pure helpers backing `GET /auth/module-identity` (see
|
||||||
|
* dbt-to-orchest docs/superpowers/specs/2026-08-20-maestro-module-identity-design.md).
|
||||||
|
*
|
||||||
|
* Kept free of HTTP/Nest so the permission mapping and tenant-routing logic
|
||||||
|
* unit-test without a request. Authorization decisions that reach orchest-api
|
||||||
|
* live on AuthClientService; this module only maps Maestro permissions to the
|
||||||
|
* Orchest header contract and derives the tenant orchest-api host.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Parse ORCHEST_ADMIN_PERMISSION_SEQIDS ("34" / "34,40"); default [34]. */
|
||||||
|
export function parseAdminSeqids(raw: string | undefined): number[] {
|
||||||
|
if (!raw || !raw.trim()) return [34];
|
||||||
|
return raw
|
||||||
|
.split(',')
|
||||||
|
.map((s) => Number(s.trim()))
|
||||||
|
.filter((n) => Number.isInteger(n));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Parse ORCHEST_MODULE_PERMISSION_SEQID; default 31 (Intelligence/Orchest). */
|
||||||
|
export function moduleGateSeqid(raw: string | undefined): number {
|
||||||
|
const n = Number(raw);
|
||||||
|
return Number.isInteger(n) && n > 0 ? n : 31;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether the user's permissions include the module-access gate seqid. */
|
||||||
|
export function isModuleAllowed(perms: number[], gateSeqid: number): boolean {
|
||||||
|
return Array.isArray(perms) && perms.includes(gateSeqid);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether the user's permissions intersect the admin seqid set. */
|
||||||
|
export function isAdmin(perms: number[], adminSeqids: number[]): boolean {
|
||||||
|
return Array.isArray(perms) && perms.some((p) => adminSeqids.includes(p));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The `{module}` slug in the tenant namespace pattern; default 'intelli'. */
|
||||||
|
export function namespaceModule(
|
||||||
|
raw: string | undefined,
|
||||||
|
): 'intelli' | 'process' {
|
||||||
|
return raw === 'process' ? 'process' : 'intelli';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The in-cluster DNS of the calling tenant's orchest-api, from the tenant
|
||||||
|
* namespace convention `orchest-{module}-{customer_name}`.
|
||||||
|
*
|
||||||
|
* `customer_name` is normalized to the namespace slug shape (lowercase,
|
||||||
|
* non-[a-z0-9-] → '-') so a display-name value still yields a valid host; a
|
||||||
|
* value that is already a clean slug is unchanged. NOTE: confirm the exact
|
||||||
|
* prod `customer_name` → namespace mapping against a real token before relying
|
||||||
|
* on this in production (the `dadosferademo2` tenant is a known exception and
|
||||||
|
* is unsupported — it is being removed).
|
||||||
|
*/
|
||||||
|
export function tenantOrchestApiHost(
|
||||||
|
customerName: string,
|
||||||
|
module: string,
|
||||||
|
): string {
|
||||||
|
const slug = String(customerName)
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9-]/g, '-');
|
||||||
|
return `orchest-api.orchest-${module}-${slug}.svc.cluster.local`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ORCHEST_API_HOST_RE =
|
||||||
|
/^orchest-api\.orchest-(intelli|process)-[a-z0-9-]+\.svc\.cluster\.local$/;
|
||||||
|
|
||||||
|
/** Defense in depth: only call an orchest-api host matching the convention. */
|
||||||
|
export function isAllowedOrchestApiHost(host: string): boolean {
|
||||||
|
return ORCHEST_API_HOST_RE.test(host);
|
||||||
|
}
|
||||||
@@ -9,7 +9,6 @@ import {
|
|||||||
Inject,
|
Inject,
|
||||||
NotFoundException,
|
NotFoundException,
|
||||||
Param,
|
Param,
|
||||||
Patch,
|
|
||||||
Post,
|
Post,
|
||||||
Put,
|
Put,
|
||||||
Query,
|
Query,
|
||||||
@@ -52,7 +51,6 @@ import {
|
|||||||
IUpdateDataRequest,
|
IUpdateDataRequest,
|
||||||
TriggerCatalogReq,
|
TriggerCatalogReq,
|
||||||
TriggerCatalogRes,
|
TriggerCatalogRes,
|
||||||
UpdateColumnsMetadataRequest,
|
|
||||||
} from './dtos';
|
} from './dtos';
|
||||||
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
|
||||||
import { Language } from 'src/decorators/language.decorator';
|
import { Language } from 'src/decorators/language.decorator';
|
||||||
@@ -450,42 +448,6 @@ export class CatalogController {
|
|||||||
return { columns_metadata };
|
return { columns_metadata };
|
||||||
}
|
}
|
||||||
|
|
||||||
@Patch('data-asset/:id/columns-metadata')
|
|
||||||
@RequireSomePermission(
|
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.UPDATE,
|
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER,
|
|
||||||
)
|
|
||||||
async updateColumnsMetadata(
|
|
||||||
@User() user: RequestUser,
|
|
||||||
@Language() language: LanguageEnum,
|
|
||||||
@Param('id') id: string,
|
|
||||||
@Body(new ValidationPipe()) body: UpdateColumnsMetadataRequest,
|
|
||||||
): Promise<{ success: boolean }> {
|
|
||||||
const { customer_name, customer_id, user_id, username } = user;
|
|
||||||
|
|
||||||
this.logger.info(`/catalog - update columns metadata`, {
|
|
||||||
user_id,
|
|
||||||
customer_name,
|
|
||||||
columns_count: body.columns.length,
|
|
||||||
});
|
|
||||||
|
|
||||||
const metadata = PackTheMetadata({
|
|
||||||
customer_name,
|
|
||||||
customer_id,
|
|
||||||
user_id,
|
|
||||||
username,
|
|
||||||
language,
|
|
||||||
});
|
|
||||||
|
|
||||||
await this.catalogService.updateColumnsDescriptions(
|
|
||||||
id,
|
|
||||||
body.columns,
|
|
||||||
metadata,
|
|
||||||
);
|
|
||||||
|
|
||||||
return { success: true };
|
|
||||||
}
|
|
||||||
|
|
||||||
@Get('data-asset/:id/preview')
|
@Get('data-asset/:id/preview')
|
||||||
@RequireSomePermission(
|
@RequireSomePermission(
|
||||||
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
|
||||||
|
|||||||
@@ -467,16 +467,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
async updateColumnsDescriptions(
|
|
||||||
id: string,
|
|
||||||
columns: { column_name: string; description: string }[],
|
|
||||||
metadata: Metadata,
|
|
||||||
) {
|
|
||||||
await lastValueFrom(
|
|
||||||
this.catalogWriteService.UpdateColumnDescriptions({ id, columns }, metadata),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) {
|
async createDataDocs(body: CreateDataDocsDTO, metadata: Metadata) {
|
||||||
if (body.asset_type === 'table' || body.asset_type === 'view') {
|
if (body.asset_type === 'table' || body.asset_type === 'view') {
|
||||||
return this.createDataDocsViaNimbus(body);
|
return this.createDataDocsViaNimbus(body);
|
||||||
@@ -842,8 +832,6 @@ class CatalogService implements OnModuleInit {
|
|||||||
data_asset_id: table_metadata_id.toString(),
|
data_asset_id: table_metadata_id.toString(),
|
||||||
customer_name: customer_name,
|
customer_name: customer_name,
|
||||||
data_asset_type: 'dataset',
|
data_asset_type: 'dataset',
|
||||||
column_metadata: [],
|
|
||||||
data_preview: '',
|
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -1,13 +1,10 @@
|
|||||||
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
|
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
|
||||||
import {
|
import {
|
||||||
ArrayNotEmpty,
|
|
||||||
IsArray,
|
|
||||||
IsEnum,
|
IsEnum,
|
||||||
IsNotEmpty,
|
IsNotEmpty,
|
||||||
|
IsOptional,
|
||||||
IsString,
|
IsString,
|
||||||
ValidateNested,
|
|
||||||
} from 'class-validator';
|
} from 'class-validator';
|
||||||
import { Type } from 'class-transformer';
|
|
||||||
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
|
||||||
|
|
||||||
export enum DataAssetShareType {
|
export enum DataAssetShareType {
|
||||||
@@ -250,26 +247,6 @@ export class IUpdateCertificationStatusRequest {
|
|||||||
certification_status: CertificationStatus;
|
certification_status: CertificationStatus;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class ColumnDescriptionDto {
|
|
||||||
@ApiProperty()
|
|
||||||
@IsString()
|
|
||||||
@IsNotEmpty()
|
|
||||||
column_name: string;
|
|
||||||
|
|
||||||
@ApiProperty()
|
|
||||||
@IsString()
|
|
||||||
description: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class UpdateColumnsMetadataRequest {
|
|
||||||
@ApiProperty({ type: [ColumnDescriptionDto] })
|
|
||||||
@IsArray()
|
|
||||||
@ArrayNotEmpty()
|
|
||||||
@ValidateNested({ each: true })
|
|
||||||
@Type(() => ColumnDescriptionDto)
|
|
||||||
columns: ColumnDescriptionDto[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ICreateDataAsset implements CreateDataAssetRequest {
|
export class ICreateDataAsset implements CreateDataAssetRequest {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
display_name: string;
|
display_name: string;
|
||||||
|
|||||||
@@ -22,8 +22,6 @@ import {
|
|||||||
ConnectionTestListTablesRes,
|
ConnectionTestListTablesRes,
|
||||||
GetTableMetadataRes,
|
GetTableMetadataRes,
|
||||||
GetTableMetadataReq,
|
GetTableMetadataReq,
|
||||||
ValidateCdcPrerequisitesReq,
|
|
||||||
ValidateCdcPrerequisitesRes,
|
|
||||||
RefreshCatalogReq,
|
RefreshCatalogReq,
|
||||||
RefreshCatalogRes,
|
RefreshCatalogRes,
|
||||||
RefreshCatalogStatusReq,
|
RefreshCatalogStatusReq,
|
||||||
@@ -131,23 +129,6 @@ export class ConnectionTestController {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Post('cdc-prerequisites')
|
|
||||||
@ApiOkResponse({ type: ValidateCdcPrerequisitesRes })
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
async validateCdcPrerequisites(
|
|
||||||
@User() user: RequestUser,
|
|
||||||
@Body(new ValidationPipe()) body: ValidateCdcPrerequisitesReq,
|
|
||||||
) {
|
|
||||||
this.logger.info('/connection-test/cdc-prerequisites', {
|
|
||||||
user: user.user_id,
|
|
||||||
customer: user.customer_name,
|
|
||||||
});
|
|
||||||
return this.connectionTestService.validateCdcPrerequisites(
|
|
||||||
body,
|
|
||||||
user.customer_name,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post('refresh-catalog')
|
@Post('refresh-catalog')
|
||||||
@ApiOkResponse({ type: RefreshCatalogRes })
|
@ApiOkResponse({ type: RefreshCatalogRes })
|
||||||
@HttpCode(HttpStatus.ACCEPTED)
|
@HttpCode(HttpStatus.ACCEPTED)
|
||||||
|
|||||||
@@ -45,24 +45,10 @@ describe('ConnectionTestService catalog cache', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('lists tables and enriches each with its cached primary keys', async () => {
|
it('keeps the existing tables response contract', async () => {
|
||||||
connectionsApiService.proxy
|
connectionsApiService.proxy.mockResolvedValue({
|
||||||
// list-tables call (names only from the catalog cache)
|
tables: [{ table_name: 'customers' }, { table_name: 'orders' }],
|
||||||
.mockResolvedValueOnce({
|
});
|
||||||
tables: [{ table_name: 'customers' }, { table_name: 'orders' }],
|
|
||||||
})
|
|
||||||
// per-table columns calls: customers has a PK, orders has none
|
|
||||||
.mockResolvedValueOnce({
|
|
||||||
columns: [
|
|
||||||
{ column_name: 'id', data_type: 'bigint', is_primary_key: true },
|
|
||||||
{ column_name: 'name', data_type: 'text', is_primary_key: false },
|
|
||||||
],
|
|
||||||
})
|
|
||||||
.mockResolvedValueOnce({
|
|
||||||
columns: [
|
|
||||||
{ column_name: 'total', data_type: 'numeric', is_primary_key: false },
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
service.connectionTestListTables(
|
service.connectionTestListTables(
|
||||||
@@ -76,14 +62,10 @@ describe('ConnectionTestService catalog cache', () => {
|
|||||||
).resolves.toEqual({
|
).resolves.toEqual({
|
||||||
operation_result: true,
|
operation_result: true,
|
||||||
table_list: ['customers', 'orders'],
|
table_list: ['customers', 'orders'],
|
||||||
tables: [
|
|
||||||
{ table_name: 'customers', primary_keys: ['id'] },
|
|
||||||
{ table_name: 'orders', primary_keys: [] },
|
|
||||||
],
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it('maps cached columns and derives references from the engine allowlist', async () => {
|
it('maps cached columns to the existing table metadata contract', async () => {
|
||||||
connectionsApiService.proxy.mockResolvedValue({
|
connectionsApiService.proxy.mockResolvedValue({
|
||||||
columns: [
|
columns: [
|
||||||
{
|
{
|
||||||
@@ -91,16 +73,6 @@ describe('ConnectionTestService catalog cache', () => {
|
|||||||
data_type: 'bigint',
|
data_type: 'bigint',
|
||||||
is_primary_key: true,
|
is_primary_key: true,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
column_name: 'name',
|
|
||||||
data_type: 'varchar',
|
|
||||||
is_primary_key: false,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
platformApiService.proxy.mockResolvedValue({
|
|
||||||
allowed_datatypes: [
|
|
||||||
{ engine: 'postgresql', allowed_datatypes: ['bigint', 'timestamp'] },
|
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -120,10 +92,13 @@ describe('ConnectionTestService catalog cache', () => {
|
|||||||
{
|
{
|
||||||
table_name: 'customers',
|
table_name: 'customers',
|
||||||
columns: [
|
columns: [
|
||||||
{ name: 'id', type: 'bigint', is_primary_key: true },
|
{
|
||||||
{ name: 'name', type: 'varchar', is_primary_key: false },
|
name: 'id',
|
||||||
|
type: 'bigint',
|
||||||
|
is_primary_key: true,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
references: [{ name: 'id', type: 'bigint', is_primary_key: true }],
|
references: [],
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
@@ -132,39 +107,6 @@ describe('ConnectionTestService catalog cache', () => {
|
|||||||
'/connection_catalog/config-id/schemas/public/tables/customers/columns',
|
'/connection_catalog/config-id/schemas/public/tables/customers/columns',
|
||||||
user,
|
user,
|
||||||
);
|
);
|
||||||
expect(platformApiService.proxy).toHaveBeenCalledWith(
|
|
||||||
'GET',
|
|
||||||
'/jobs/jdbc/configs/allowed_datatypes',
|
|
||||||
user,
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('returns empty references when the allowlist call fails', async () => {
|
|
||||||
connectionsApiService.proxy.mockResolvedValue({
|
|
||||||
columns: [{ column_name: 'id', data_type: 'bigint', is_primary_key: true }],
|
|
||||||
});
|
|
||||||
platformApiService.proxy.mockRejectedValue(new Error('platform down'));
|
|
||||||
|
|
||||||
await expect(
|
|
||||||
service.getTableMetadata(
|
|
||||||
{
|
|
||||||
connection_id: 'config-id',
|
|
||||||
plugin: 'postgresql',
|
|
||||||
schema: 'public',
|
|
||||||
table_list: ['customers'],
|
|
||||||
},
|
|
||||||
user,
|
|
||||||
),
|
|
||||||
).resolves.toEqual({
|
|
||||||
operation_result: true,
|
|
||||||
tables_metadata: [
|
|
||||||
{
|
|
||||||
table_name: 'customers',
|
|
||||||
columns: [{ name: 'id', type: 'bigint', is_primary_key: true }],
|
|
||||||
references: [],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it('submits a catalog refresh without holding the request open', async () => {
|
it('submits a catalog refresh without holding the request open', async () => {
|
||||||
|
|||||||
@@ -13,8 +13,6 @@ import {
|
|||||||
ConnectionTestPingRes,
|
ConnectionTestPingRes,
|
||||||
GetTableMetadataReq,
|
GetTableMetadataReq,
|
||||||
GetTableMetadataRes,
|
GetTableMetadataRes,
|
||||||
ValidateCdcPrerequisitesReq,
|
|
||||||
ValidateCdcPrerequisitesRes,
|
|
||||||
RefreshCatalogReq,
|
RefreshCatalogReq,
|
||||||
RefreshCatalogRes,
|
RefreshCatalogRes,
|
||||||
RefreshCatalogStatusReq,
|
RefreshCatalogStatusReq,
|
||||||
@@ -179,33 +177,9 @@ export class ConnectionTestService {
|
|||||||
`/schemas/${encodeURIComponent(body.schema)}/tables`,
|
`/schemas/${encodeURIComponent(body.schema)}/tables`,
|
||||||
user,
|
user,
|
||||||
);
|
);
|
||||||
const table_names: string[] = result.tables.map((table) => table.table_name);
|
|
||||||
// CDC create needs the primary keys per table (used to build the deduped
|
|
||||||
// Iceberg identifier-fields). The catalog-cache list-tables endpoint returns
|
|
||||||
// only names, so fetch each table's columns from the cache and keep the ones
|
|
||||||
// flagged is_primary_key. Reads hit the stored catalog snapshot (populated by
|
|
||||||
// refresh-catalog), never the live connection.
|
|
||||||
const tables = await Promise.all(
|
|
||||||
table_names.map(async (table_name) => {
|
|
||||||
const columns = await this.connectionsApiService.proxy(
|
|
||||||
'GET',
|
|
||||||
`/connection_catalog/${encodeURIComponent(body.connection_id)}` +
|
|
||||||
`/schemas/${encodeURIComponent(body.schema)}` +
|
|
||||||
`/tables/${encodeURIComponent(table_name)}/columns`,
|
|
||||||
user,
|
|
||||||
);
|
|
||||||
return {
|
|
||||||
table_name,
|
|
||||||
primary_keys: columns.columns
|
|
||||||
.filter((column) => column.is_primary_key)
|
|
||||||
.map((column) => column.column_name),
|
|
||||||
};
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
return {
|
return {
|
||||||
operation_result: true,
|
operation_result: true,
|
||||||
table_list: table_names,
|
table_list: result.tables.map((table) => table.table_name),
|
||||||
tables,
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -213,20 +187,6 @@ export class ConnectionTestService {
|
|||||||
body: GetTableMetadataReq,
|
body: GetTableMetadataReq,
|
||||||
user: RequestUser,
|
user: RequestUser,
|
||||||
): Promise<GetTableMetadataRes> {
|
): Promise<GetTableMetadataRes> {
|
||||||
// Columns eligible as the incremental reference field are the ones whose
|
|
||||||
// data type is allowed for this engine (e.g. int/date/timestamp). The
|
|
||||||
// allowlist is owned by the platform API, keyed by engine === plugin.
|
|
||||||
const allowedByEngine = await this.platformApiService
|
|
||||||
.proxy('GET', '/jobs/jdbc/configs/allowed_datatypes', user)
|
|
||||||
.catch(() => null);
|
|
||||||
const allowedDataTypes: string[] =
|
|
||||||
allowedByEngine?.allowed_datatypes?.find(
|
|
||||||
(datatypes) => datatypes.engine === body.plugin,
|
|
||||||
)?.allowed_datatypes ?? [];
|
|
||||||
const allowedSet = new Set(
|
|
||||||
allowedDataTypes.map((type) => type.toLowerCase()),
|
|
||||||
);
|
|
||||||
|
|
||||||
const tables_metadata = await Promise.all(
|
const tables_metadata = await Promise.all(
|
||||||
body.table_list.map(async (table_name) => {
|
body.table_list.map(async (table_name) => {
|
||||||
const result = await this.connectionsApiService.proxy(
|
const result = await this.connectionsApiService.proxy(
|
||||||
@@ -236,18 +196,14 @@ export class ConnectionTestService {
|
|||||||
`/tables/${encodeURIComponent(table_name)}/columns`,
|
`/tables/${encodeURIComponent(table_name)}/columns`,
|
||||||
user,
|
user,
|
||||||
);
|
);
|
||||||
const columns = result.columns.map((column) => ({
|
|
||||||
name: column.column_name,
|
|
||||||
type: column.data_type,
|
|
||||||
is_primary_key: column.is_primary_key,
|
|
||||||
}));
|
|
||||||
const references = columns.filter((column) =>
|
|
||||||
allowedSet.has(String(column.type).toLowerCase()),
|
|
||||||
);
|
|
||||||
return {
|
return {
|
||||||
table_name,
|
table_name,
|
||||||
columns,
|
columns: result.columns.map((column) => ({
|
||||||
references,
|
name: column.column_name,
|
||||||
|
type: column.data_type,
|
||||||
|
is_primary_key: column.is_primary_key,
|
||||||
|
})),
|
||||||
|
references: [],
|
||||||
};
|
};
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
@@ -331,18 +287,4 @@ export class ConnectionTestService {
|
|||||||
date: body.date,
|
date: body.date,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async validateCdcPrerequisites(
|
|
||||||
body: ValidateCdcPrerequisitesReq,
|
|
||||||
customer_name: string,
|
|
||||||
): Promise<ValidateCdcPrerequisitesRes> {
|
|
||||||
const { plugin, connection_id } = body;
|
|
||||||
return lastValueFrom(
|
|
||||||
this.connectionTestReadClient.ValidateCdcPrerequisites({
|
|
||||||
connection_id,
|
|
||||||
customer_name,
|
|
||||||
plugin,
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -102,20 +102,11 @@ export class ConnectionTestListTablesReq {
|
|||||||
schema: string;
|
schema: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class ConnectionTestListTablesEntry {
|
|
||||||
@ApiProperty()
|
|
||||||
table_name: string;
|
|
||||||
@ApiProperty({ type: [String] })
|
|
||||||
primary_keys: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ConnectionTestListTablesRes {
|
export class ConnectionTestListTablesRes {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
operation_result: boolean;
|
operation_result: boolean;
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
table_list: string[];
|
table_list: string[];
|
||||||
@ApiProperty({ type: [ConnectionTestListTablesEntry] })
|
|
||||||
tables: ConnectionTestListTablesEntry[];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export class GetTableMetadataReq {
|
export class GetTableMetadataReq {
|
||||||
@@ -143,59 +134,13 @@ export class GetTableMetadataRes {
|
|||||||
tables_metadata: TableMetadataDto[];
|
tables_metadata: TableMetadataDto[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export class CdcCheckDto {
|
|
||||||
@ApiProperty()
|
|
||||||
name: string;
|
|
||||||
@ApiProperty()
|
|
||||||
expected: string;
|
|
||||||
@ApiProperty()
|
|
||||||
actual: string;
|
|
||||||
@ApiProperty()
|
|
||||||
passed: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ValidateCdcPrerequisitesReq {
|
|
||||||
@ApiProperty()
|
|
||||||
@IsString()
|
|
||||||
plugin: string;
|
|
||||||
|
|
||||||
@ApiProperty()
|
|
||||||
@IsString()
|
|
||||||
connection_id: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ValidateCdcPrerequisitesRes {
|
|
||||||
@ApiProperty()
|
|
||||||
operation_result: boolean;
|
|
||||||
@ApiProperty({ type: [CdcCheckDto] })
|
|
||||||
checks: CdcCheckDto[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export class RefreshCatalogReq {
|
export class RefreshCatalogReq {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
@IsString()
|
@IsString()
|
||||||
connection_id: string;
|
connection_id: string;
|
||||||
|
|
||||||
@ApiProperty({
|
@ApiProperty({ enum: ['oracle', 'mysql', 'postgresql', 'sqlserver'] })
|
||||||
enum: [
|
@IsIn(['oracle', 'mysql', 'postgresql', 'sqlserver'])
|
||||||
'oracle',
|
|
||||||
'mysql',
|
|
||||||
'postgresql',
|
|
||||||
'sqlserver',
|
|
||||||
'mysql_cdc',
|
|
||||||
'postgresql_cdc',
|
|
||||||
'oracle_cdc',
|
|
||||||
],
|
|
||||||
})
|
|
||||||
@IsIn([
|
|
||||||
'oracle',
|
|
||||||
'mysql',
|
|
||||||
'postgresql',
|
|
||||||
'sqlserver',
|
|
||||||
'mysql_cdc',
|
|
||||||
'postgresql_cdc',
|
|
||||||
'oracle_cdc',
|
|
||||||
])
|
|
||||||
plugin: string;
|
plugin: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,43 +0,0 @@
|
|||||||
import {
|
|
||||||
CdcColumn,
|
|
||||||
CdcTable,
|
|
||||||
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
|
||||||
|
|
||||||
/** What a caller knows about a CDC table before it is stored. The source
|
|
||||||
* table is named `table_name` on the platform-facing bodies and `name` on
|
|
||||||
* the create DTO (CdcTableReq); either works — deriving one from the other
|
|
||||||
* happens only here. */
|
|
||||||
export interface CdcTableInput {
|
|
||||||
// Optional on the create DTO (CdcTableReq); the platform validates it.
|
|
||||||
table_schema?: string;
|
|
||||||
table_name?: string;
|
|
||||||
name?: string;
|
|
||||||
primary_keys?: string[];
|
|
||||||
iceberg_table_name?: string;
|
|
||||||
iceberg_qualify_table_name?: string;
|
|
||||||
columns?: CdcColumn[];
|
|
||||||
column_exclude_list?: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The only place maestro builds a protospack `CdcTable`.
|
|
||||||
*
|
|
||||||
* `name` is the identity key shared with batch `NewTable` (in-factory keys
|
|
||||||
* update/soft-delete on it) and the proto keeps it required, so it mirrors
|
|
||||||
* `table_name` here and nowhere else. Follow-up (protospack): make `name`
|
|
||||||
* optional and let in-factory be its sole writer (it already backfills
|
|
||||||
* `name ?? table_name`).
|
|
||||||
*/
|
|
||||||
export function toCdcTable(table: CdcTableInput): CdcTable {
|
|
||||||
const tableName = table.table_name ?? table.name;
|
|
||||||
return {
|
|
||||||
table_schema: table.table_schema,
|
|
||||||
table_name: tableName,
|
|
||||||
name: tableName,
|
|
||||||
primary_keys: table.primary_keys ?? [],
|
|
||||||
iceberg_table_name: table.iceberg_table_name,
|
|
||||||
iceberg_qualify_table_name: table.iceberg_qualify_table_name,
|
|
||||||
columns: table.columns ?? [],
|
|
||||||
column_exclude_list: table.column_exclude_list ?? [],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -65,64 +65,3 @@ export class CreateInputReq extends OmitType(Input, [
|
|||||||
'created_at',
|
'created_at',
|
||||||
'updated_at',
|
'updated_at',
|
||||||
]) {}
|
]) {}
|
||||||
|
|
||||||
export class CdcColumnReq {
|
|
||||||
@ApiProperty()
|
|
||||||
name: string;
|
|
||||||
@ApiProperty()
|
|
||||||
type: string;
|
|
||||||
@ApiProperty()
|
|
||||||
is_primary_key: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CdcTableReq {
|
|
||||||
@ApiProperty()
|
|
||||||
name: string;
|
|
||||||
// Source database/schema. Required: Debezium addresses tables as
|
|
||||||
// `schema.table`, a CDC table without it cannot be replicated.
|
|
||||||
@ApiProperty()
|
|
||||||
table_schema: string;
|
|
||||||
@ApiPropertyOptional({ type: [String] })
|
|
||||||
primary_keys?: string[];
|
|
||||||
// Per-table raw Iceberg table name override (iceberg destination only).
|
|
||||||
// Honored on the create/add path: the platform lowercases + sanitizes it
|
|
||||||
// authoritatively; empty/absent => the platform derives tb__<hash>__<table>.
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
iceberg_table_name?: string;
|
|
||||||
// Per-table deduped (qualify) Iceberg table name override (iceberg dest only).
|
|
||||||
// Empty/absent => the deduped table takes the same name as the raw table.
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
iceberg_qualify_table_name?: string;
|
|
||||||
@ApiPropertyOptional({ type: [CdcColumnReq] })
|
|
||||||
columns?: CdcColumnReq[];
|
|
||||||
// Columns the user chose to ignore -> Debezium column.exclude.list.
|
|
||||||
@ApiPropertyOptional({ type: [String] })
|
|
||||||
column_exclude_list?: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export class IcebergDestinationReq {
|
|
||||||
@ApiProperty()
|
|
||||||
namespace: string;
|
|
||||||
// Pipeline-wide deduped (qualify) namespace. Absent => the platform derives
|
|
||||||
// the sibling of `namespace` (cdc_raw -> cdc_dedup).
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
qualify_namespace?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CdcDestinationReq {
|
|
||||||
@ApiPropertyOptional({ type: IcebergDestinationReq })
|
|
||||||
iceberg?: IcebergDestinationReq;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CreateCdcInputReq {
|
|
||||||
@ApiProperty()
|
|
||||||
name: string;
|
|
||||||
@ApiProperty()
|
|
||||||
plugin: string; // mysql_cdc (v1)
|
|
||||||
@ApiProperty({ type: [CdcTableReq] })
|
|
||||||
tables: CdcTableReq[];
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
read_only?: boolean;
|
|
||||||
@ApiPropertyOptional({ type: CdcDestinationReq })
|
|
||||||
destination?: CdcDestinationReq;
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,75 +0,0 @@
|
|||||||
import { Test, TestingModule } from '@nestjs/testing';
|
|
||||||
import { InputsController } from './inputs.controller';
|
|
||||||
import { InputsService } from './inputs.service';
|
|
||||||
import DadosferaLogger from '@dadosfera/dadosfera-logs';
|
|
||||||
import { CreateCdcInputReq } from './dtos/input.model';
|
|
||||||
import { RequestUser } from 'src/decorators/user.decorator';
|
|
||||||
|
|
||||||
describe('InputsController', () => {
|
|
||||||
let controller: InputsController;
|
|
||||||
let inputsService: { createCdc: jest.Mock };
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
inputsService = {
|
|
||||||
createCdc: jest.fn().mockResolvedValue({ input: {} }),
|
|
||||||
};
|
|
||||||
|
|
||||||
const module: TestingModule = await Test.createTestingModule({
|
|
||||||
controllers: [InputsController],
|
|
||||||
providers: [
|
|
||||||
{
|
|
||||||
provide: DadosferaLogger,
|
|
||||||
useValue: { logger: { info: jest.fn() } },
|
|
||||||
},
|
|
||||||
{
|
|
||||||
provide: InputsService,
|
|
||||||
useValue: inputsService,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}).compile();
|
|
||||||
|
|
||||||
controller = module.get<InputsController>(InputsController);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should be defined', () => {
|
|
||||||
expect(controller).toBeDefined();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('forwards destination.iceberg.namespace to InputsService.createCdc', async () => {
|
|
||||||
const body: CreateCdcInputReq = {
|
|
||||||
name: 'my-cdc-input',
|
|
||||||
plugin: 'mysql_cdc',
|
|
||||||
tables: [
|
|
||||||
{
|
|
||||||
name: 'orders',
|
|
||||||
table_schema: 'public',
|
|
||||||
iceberg_table_name: 'orders_iceberg',
|
|
||||||
},
|
|
||||||
],
|
|
||||||
destination: {
|
|
||||||
iceberg: {
|
|
||||||
namespace: 'my_namespace',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
const user: RequestUser = {
|
|
||||||
user_id: 'user-1',
|
|
||||||
customer_id: 'customer-1',
|
|
||||||
customer_name: 'customer',
|
|
||||||
} as RequestUser;
|
|
||||||
|
|
||||||
await controller.createCdc(body, user);
|
|
||||||
|
|
||||||
expect(inputsService.createCdc).toHaveBeenCalledWith(
|
|
||||||
expect.objectContaining({
|
|
||||||
body: expect.objectContaining({
|
|
||||||
destination: {
|
|
||||||
iceberg: {
|
|
||||||
namespace: 'my_namespace',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}),
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -15,7 +15,6 @@ import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
|
|||||||
import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
|
import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
|
||||||
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
|
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
|
||||||
import {
|
import {
|
||||||
CreateCdcInputReq,
|
|
||||||
CreateInputReq,
|
CreateInputReq,
|
||||||
GetAvailableEntitiesReq,
|
GetAvailableEntitiesReq,
|
||||||
GetAvailableEntitiesRes,
|
GetAvailableEntitiesRes,
|
||||||
@@ -106,26 +105,6 @@ export class InputsController {
|
|||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Post('cdc')
|
|
||||||
@ApiInternalOnlyEndpoint()
|
|
||||||
@ApiOkResponse({ type: Input })
|
|
||||||
async createCdc(
|
|
||||||
@Body() body: CreateCdcInputReq,
|
|
||||||
@User() user: RequestUser,
|
|
||||||
) {
|
|
||||||
const info: Info = {
|
|
||||||
user_id: user.user_id,
|
|
||||||
customer: user.customer_name,
|
|
||||||
customer_id: user.customer_id,
|
|
||||||
};
|
|
||||||
this.logger.info(`/inputs/cdc - ON CREATE CDC INPUT ROUTE`, {
|
|
||||||
user: info.user_id,
|
|
||||||
customer: info.customer,
|
|
||||||
});
|
|
||||||
|
|
||||||
return this.inputService.createCdc({ body, info });
|
|
||||||
}
|
|
||||||
|
|
||||||
@ApiInternalOnlyEndpoint()
|
@ApiInternalOnlyEndpoint()
|
||||||
@Get()
|
@Get()
|
||||||
async findAll(@User() user: RequestUser) {
|
async findAll(@User() user: RequestUser) {
|
||||||
|
|||||||
@@ -1,113 +0,0 @@
|
|||||||
import { of } from 'rxjs';
|
|
||||||
import { InputsService } from './inputs.service';
|
|
||||||
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
|
|
||||||
import { CreateCdcInputReq } from './dtos/input.model';
|
|
||||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
|
||||||
|
|
||||||
const info = { customer_id: 'cid', user_id: 'u' } as unknown as Info;
|
|
||||||
|
|
||||||
describe('InputsService.createCdc', () => {
|
|
||||||
let service: InputsService;
|
|
||||||
let inputCreateCdcMock: jest.Mock;
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
inputCreateCdcMock = jest
|
|
||||||
.fn()
|
|
||||||
.mockImplementation((req) => of({ input: req.input }));
|
|
||||||
|
|
||||||
const grpcClient: any = {
|
|
||||||
getService: jest.fn().mockReturnValue({
|
|
||||||
InputCreateCdc: inputCreateCdcMock,
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
|
|
||||||
service = new InputsService(new DadosferaLogger(), grpcClient);
|
|
||||||
await service.onModuleInit();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('forwards destination and per-table iceberg_table_name to the gRPC request', async () => {
|
|
||||||
const body: CreateCdcInputReq = {
|
|
||||||
name: 'CDC Iceberg Test',
|
|
||||||
plugin: 'mysql_cdc',
|
|
||||||
read_only: true,
|
|
||||||
destination: { iceberg: { namespace: 'cdc_raw' } },
|
|
||||||
tables: [
|
|
||||||
{
|
|
||||||
name: 'orders',
|
|
||||||
table_schema: 'mydb',
|
|
||||||
primary_keys: ['id'],
|
|
||||||
iceberg_table_name: 'cdc_raw.mydb__orders',
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
|
|
||||||
await service.createCdc({ body, info });
|
|
||||||
|
|
||||||
expect(inputCreateCdcMock).toHaveBeenCalledTimes(1);
|
|
||||||
const sentRequest = inputCreateCdcMock.mock.calls[0][0];
|
|
||||||
|
|
||||||
expect(sentRequest.input).toEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
destination: { iceberg: { namespace: 'cdc_raw' } },
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
expect(sentRequest.input.tables[0]).toEqual(
|
|
||||||
expect.objectContaining({
|
|
||||||
iceberg_table_name: 'cdc_raw.mydb__orders',
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('forwards per-table columns to the gRPC request', async () => {
|
|
||||||
const body: CreateCdcInputReq = {
|
|
||||||
name: 'CDC Columns Test',
|
|
||||||
plugin: 'mysql_cdc',
|
|
||||||
read_only: true,
|
|
||||||
tables: [
|
|
||||||
{
|
|
||||||
name: 'orders',
|
|
||||||
table_schema: 'mydb',
|
|
||||||
primary_keys: ['id'],
|
|
||||||
columns: [
|
|
||||||
{ name: 'id', type: 'int', is_primary_key: true },
|
|
||||||
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
|
|
||||||
await service.createCdc({ body, info });
|
|
||||||
|
|
||||||
expect(inputCreateCdcMock).toHaveBeenCalledTimes(1);
|
|
||||||
const sentRequest = inputCreateCdcMock.mock.calls[0][0];
|
|
||||||
|
|
||||||
expect(sentRequest.input.tables[0].columns).toEqual([
|
|
||||||
{ name: 'id', type: 'int', is_primary_key: true },
|
|
||||||
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
|
|
||||||
]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('back-compat: a body with no destination sends destination undefined, not an error', async () => {
|
|
||||||
const body: CreateCdcInputReq = {
|
|
||||||
name: 'CDC Legacy Test',
|
|
||||||
plugin: 'mysql_cdc',
|
|
||||||
read_only: true,
|
|
||||||
tables: [
|
|
||||||
{
|
|
||||||
name: 'pedidos',
|
|
||||||
table_schema: 'cadastros',
|
|
||||||
primary_keys: ['id'],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
};
|
|
||||||
|
|
||||||
const result = await service.createCdc({ body, info });
|
|
||||||
|
|
||||||
expect(inputCreateCdcMock).toHaveBeenCalledTimes(1);
|
|
||||||
const sentRequest = inputCreateCdcMock.mock.calls[0][0];
|
|
||||||
|
|
||||||
expect(sentRequest.input.destination).toBeUndefined();
|
|
||||||
expect(sentRequest.input.tables[0].iceberg_table_name).toBeUndefined();
|
|
||||||
expect(result.input).toBeDefined();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -13,21 +13,16 @@ import { objectCamelToSnake } from 'src/utils/CaseConverter';
|
|||||||
import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
|
import { IIdRequest, UpdateInputRequest } from './dtos/old_interfaces';
|
||||||
import { Input } from '@dadosfera/protospack-v2';
|
import { Input } from '@dadosfera/protospack-v2';
|
||||||
import {
|
import {
|
||||||
AddCdcTableRequest,
|
|
||||||
GetAvailableEntitiesRequest,
|
GetAvailableEntitiesRequest,
|
||||||
InputCreateGenericRequest,
|
InputCreateGenericRequest,
|
||||||
InputCreateCdcRequest,
|
|
||||||
InputCreateS3Request,
|
InputCreateS3Request,
|
||||||
InputNewCreateRequest,
|
InputNewCreateRequest,
|
||||||
InputUpdateResponse,
|
InputUpdateResponse,
|
||||||
MarkTableDeletedRequest,
|
|
||||||
RemoveCdcTableRequest,
|
|
||||||
RollbackInputRequest,
|
RollbackInputRequest,
|
||||||
TestConnectionRequest,
|
TestConnectionRequest,
|
||||||
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
} from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/messages';
|
||||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
||||||
import { CreateCdcInputReq, CreateInputReq } from './dtos/input.model';
|
import { CreateInputReq } from './dtos/input.model';
|
||||||
import { toCdcTable } from './cdc-table.mapper';
|
|
||||||
import { Metadata } from '@grpc/grpc-js';
|
import { Metadata } from '@grpc/grpc-js';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
@@ -188,26 +183,6 @@ export class InputsService {
|
|||||||
return { input: adjustedInput };
|
return { input: adjustedInput };
|
||||||
}
|
}
|
||||||
|
|
||||||
async createCdc(data: { body: CreateCdcInputReq; info: Info }) {
|
|
||||||
const { body, info } = data;
|
|
||||||
|
|
||||||
const inputCreateCdcRequest: InputCreateCdcRequest = {
|
|
||||||
input: {
|
|
||||||
name: body.name,
|
|
||||||
plugin: body.plugin,
|
|
||||||
read_only: body.read_only ?? true,
|
|
||||||
tables: body.tables.map(toCdcTable),
|
|
||||||
destination: body.destination,
|
|
||||||
},
|
|
||||||
info,
|
|
||||||
};
|
|
||||||
|
|
||||||
const { input } = await lastValueFrom(
|
|
||||||
this.inputWriteService.InputCreateCdc(inputCreateCdcRequest),
|
|
||||||
);
|
|
||||||
return { input };
|
|
||||||
}
|
|
||||||
|
|
||||||
async getAvailableEntities(data: GetAvailableEntitiesRequest) {
|
async getAvailableEntities(data: GetAvailableEntitiesRequest) {
|
||||||
return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
|
return lastValueFrom(this.inputReadService.GetAvailableEntities(data));
|
||||||
}
|
}
|
||||||
@@ -315,19 +290,11 @@ export class InputsService {
|
|||||||
return formatedPayload;
|
return formatedPayload;
|
||||||
}
|
}
|
||||||
|
|
||||||
async markTableDeleted(data: MarkTableDeletedRequest) {
|
async markTableDeleted(data: { input_id: string; table_name: string; info: Info }) {
|
||||||
return lastValueFrom(this.inputWriteService.MarkTableDeleted(data));
|
return lastValueFrom(this.inputWriteService.MarkTableDeleted(data));
|
||||||
}
|
}
|
||||||
|
|
||||||
async unmarkTableDeleted(data: MarkTableDeletedRequest) {
|
async unmarkTableDeleted(data: { input_id: string; table_name: string; info: Info }) {
|
||||||
return lastValueFrom(this.inputWriteService.UnmarkTableDeleted(data));
|
return lastValueFrom((this.inputWriteService as any).UnmarkTableDeleted(data));
|
||||||
}
|
|
||||||
|
|
||||||
async addCdcTable(data: AddCdcTableRequest) {
|
|
||||||
return lastValueFrom(this.inputWriteService.AddCdcTable(data));
|
|
||||||
}
|
|
||||||
|
|
||||||
async removeCdcTable(data: RemoveCdcTableRequest) {
|
|
||||||
return lastValueFrom(this.inputWriteService.RemoveCdcTable(data));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,33 +9,6 @@ export class PipelineInputsDTO {
|
|||||||
}>
|
}>
|
||||||
}
|
}
|
||||||
|
|
||||||
export class PipelineTableDestination {
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
table_schema?: string;
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
table_name?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** One entry of the create body's `config.tables`. Mirrors the frontend's
|
|
||||||
* EntityWithColumnsNames; only the fields maestro/pi-factory read are typed,
|
|
||||||
* the rest travels as-is. */
|
|
||||||
export class PipelineTableConfig {
|
|
||||||
@ApiProperty()
|
|
||||||
name: string;
|
|
||||||
@ApiPropertyOptional({ type: () => PipelineTableDestination })
|
|
||||||
destinations?: Partial<Record<'raw' | 'qualify', PipelineTableDestination>>;
|
|
||||||
[extra: string]: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** `{ cron, tables }` as sent by the frontend on create; JSON-serialized onto
|
|
||||||
* the gRPC `config` string field. */
|
|
||||||
export class PipelineConfig {
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
cron?: string;
|
|
||||||
@ApiPropertyOptional({ type: [PipelineTableConfig] })
|
|
||||||
tables?: PipelineTableConfig[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export class IPipelineV2 {
|
export class IPipelineV2 {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
id: string;
|
id: string;
|
||||||
@@ -58,8 +31,6 @@ export class IPipelineV2 {
|
|||||||
tags?: string[];
|
tags?: string[];
|
||||||
@ApiPropertyOptional()
|
@ApiPropertyOptional()
|
||||||
properties?: any;
|
properties?: any;
|
||||||
@ApiPropertyOptional({ type: () => PipelineConfig })
|
|
||||||
config?: PipelineConfig;
|
|
||||||
|
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
connector_name: string;
|
connector_name: string;
|
||||||
|
|||||||
@@ -25,10 +25,6 @@ export class PipelinesClientConfiguration {
|
|||||||
loader: {
|
loader: {
|
||||||
keepCase: true,
|
keepCase: true,
|
||||||
enums: String,
|
enums: String,
|
||||||
// int64 fields (PipelineV2SinkTableOffset.committed_offset) decode as
|
|
||||||
// plain JS numbers instead of Long.js objects, so they serialize as
|
|
||||||
// JSON numbers for the frontend. Safe: Kafka offsets fit in 2^53.
|
|
||||||
longs: Number,
|
|
||||||
objects: true,
|
objects: true,
|
||||||
arrays: true,
|
arrays: true,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -510,78 +510,4 @@ export class PipelinesController {
|
|||||||
|
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- CDC pipeline operations (Kafka Connect backed) ----
|
|
||||||
// These operate on an existing pipeline, so they require UPDATE (not CREATE).
|
|
||||||
|
|
||||||
@Get(':id/live-status')
|
|
||||||
@ApiInternalOnlyEndpoint()
|
|
||||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
|
||||||
async getLiveStatus(
|
|
||||||
@Param('id') id: string,
|
|
||||||
@User() user: RequestUser,
|
|
||||||
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
|
|
||||||
this.logger.info('PipelinesController - getLiveStatus', { id });
|
|
||||||
const metadata = PackTheMetadata(user);
|
|
||||||
return this.pipelinesClientService.getLiveStatus(id, metadata);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post(':id/pause')
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
@ApiInternalOnlyEndpoint()
|
|
||||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
|
||||||
async pause(
|
|
||||||
@Param('id') id: string,
|
|
||||||
@User() user: RequestUser,
|
|
||||||
): Promise<Messages.PipelineV2OperationResponse> {
|
|
||||||
this.logger.info('PipelinesController - pause', { id });
|
|
||||||
const metadata = PackTheMetadata(user);
|
|
||||||
return this.pipelinesClientService.pause(id, metadata);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post(':id/unpause')
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
@ApiInternalOnlyEndpoint()
|
|
||||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
|
||||||
async unpause(
|
|
||||||
@Param('id') id: string,
|
|
||||||
@User() user: RequestUser,
|
|
||||||
): Promise<Messages.PipelineV2OperationResponse> {
|
|
||||||
this.logger.info('PipelinesController - unpause', { id });
|
|
||||||
const metadata = PackTheMetadata(user);
|
|
||||||
return this.pipelinesClientService.unpause(id, metadata);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post(':id/restart')
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
@ApiInternalOnlyEndpoint()
|
|
||||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
|
||||||
async restart(
|
|
||||||
@Param('id') id: string,
|
|
||||||
@User() user: RequestUser,
|
|
||||||
): Promise<Messages.PipelineV2OperationResponse> {
|
|
||||||
this.logger.info('PipelinesController - restart', { id });
|
|
||||||
const metadata = PackTheMetadata(user);
|
|
||||||
return this.pipelinesClientService.restart(id, metadata);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post(':id/jobs/:jobId/reset-state')
|
|
||||||
@HttpCode(HttpStatus.OK)
|
|
||||||
@ApiInternalOnlyEndpoint()
|
|
||||||
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
|
||||||
async resetJobState(
|
|
||||||
@Param('id') id: string,
|
|
||||||
@Param('jobId') jobId: string,
|
|
||||||
@Body() body: { schedule_minutes?: number },
|
|
||||||
@User() user: RequestUser,
|
|
||||||
): Promise<Messages.PipelineV2OperationResponse> {
|
|
||||||
this.logger.info('PipelinesController - resetJobState', { id, jobId });
|
|
||||||
const metadata = PackTheMetadata(user);
|
|
||||||
return this.pipelinesClientService.resetJobState(
|
|
||||||
id,
|
|
||||||
jobId,
|
|
||||||
body?.schedule_minutes,
|
|
||||||
metadata,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Module, forwardRef } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
import { ClientsModule } from '@nestjs/microservices';
|
import { ClientsModule } from '@nestjs/microservices';
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
|
||||||
@@ -23,7 +23,7 @@ const client = new PipelinesClientConfiguration();
|
|||||||
ConnectorModule,
|
ConnectorModule,
|
||||||
InputsModule,
|
InputsModule,
|
||||||
TransformationsModule,
|
TransformationsModule,
|
||||||
forwardRef(() => PlatformApiModule),
|
PlatformApiModule,
|
||||||
NimbusServicesModule,
|
NimbusServicesModule,
|
||||||
CatalogModule
|
CatalogModule
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -1,142 +0,0 @@
|
|||||||
// These imported modules pull in gRPC client-config / service modules that read
|
|
||||||
// process.env at load time; mock them (hoisted before imports) so the spec needs
|
|
||||||
// no runtime env. Each mock severs an entire import subtree while still providing
|
|
||||||
// a class usable as a value/DI token. Mirrors platform-api.controller.spec.ts.
|
|
||||||
jest.mock('./pipelines-client', () => ({ PipelinesClientConfiguration: class {} }));
|
|
||||||
jest.mock('../connector/client.service', () => ({ ConnectorClientService: class {} }));
|
|
||||||
jest.mock('../inputs/inputs.service', () => ({ InputsService: class {} }));
|
|
||||||
jest.mock('../transformations/transformations.service', () => ({ TransformationsService: class {} }));
|
|
||||||
jest.mock('../platform-api/platform-api.service', () => ({ PlatformApiService: class {} }));
|
|
||||||
jest.mock('src/services/nimbus/nimbus.service', () => ({ NimbusService: class {} }));
|
|
||||||
jest.mock('../catalog/catalog.service', () => ({ CatalogService: class {} }));
|
|
||||||
|
|
||||||
import { of } from 'rxjs';
|
|
||||||
import { PipelinesService } from './pipelines.service';
|
|
||||||
|
|
||||||
const logger = {
|
|
||||||
info: jest.fn(),
|
|
||||||
error: jest.fn(),
|
|
||||||
};
|
|
||||||
|
|
||||||
const cdcOldInput = { input: { plugin: 'mysql_cdc', tables: [] } };
|
|
||||||
const batchOldInput = { input: { plugin: 'mysql', type: 'database', tables: [] } };
|
|
||||||
|
|
||||||
const updateResponse = {
|
|
||||||
input: { type: 'database' },
|
|
||||||
tablesUpdate: [],
|
|
||||||
dataAssetUpdate: [],
|
|
||||||
};
|
|
||||||
|
|
||||||
const user: any = { customer_modules: [] };
|
|
||||||
const updateInputDTO: any = { tables: [] };
|
|
||||||
const info: any = { customer: 'cust' };
|
|
||||||
const metadata: any = {};
|
|
||||||
|
|
||||||
function buildService(oldInput: any) {
|
|
||||||
const inputsService: any = {
|
|
||||||
findOne: jest.fn().mockResolvedValue(oldInput),
|
|
||||||
update: jest.fn().mockResolvedValue(updateResponse),
|
|
||||||
rollbackUpdate: jest.fn().mockResolvedValue({}),
|
|
||||||
};
|
|
||||||
const nimbusService: any = { renameTable: jest.fn().mockResolvedValue({}) };
|
|
||||||
|
|
||||||
const service = new PipelinesService(
|
|
||||||
{ logger } as any, // dadosferaLogger
|
|
||||||
{} as any, // grpcClient
|
|
||||||
{} as any, // connectorService
|
|
||||||
inputsService, // inputsService
|
|
||||||
{} as any, // transformationsService
|
|
||||||
{} as any, // platformAPI
|
|
||||||
nimbusService, // nimbusService
|
|
||||||
{} as any, // catalogService
|
|
||||||
);
|
|
||||||
|
|
||||||
const updatePlatformJobsSpy = jest
|
|
||||||
.spyOn(service, 'updatePlatformJobs')
|
|
||||||
.mockResolvedValue(undefined as any);
|
|
||||||
|
|
||||||
return { service, inputsService, updatePlatformJobsSpy };
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('PipelinesService - updatePipelineInput', () => {
|
|
||||||
afterEach(() => jest.clearAllMocks());
|
|
||||||
|
|
||||||
it('CDC input skips updatePlatformJobs', async () => {
|
|
||||||
const { service, inputsService, updatePlatformJobsSpy } = buildService(cdcOldInput);
|
|
||||||
|
|
||||||
const result = await service.updatePipelineInput(
|
|
||||||
'pipeline-id',
|
|
||||||
'input-id',
|
|
||||||
updateInputDTO,
|
|
||||||
info,
|
|
||||||
user,
|
|
||||||
metadata,
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(updatePlatformJobsSpy).not.toHaveBeenCalled();
|
|
||||||
expect(inputsService.update).toHaveBeenCalled();
|
|
||||||
expect(result).toBe(updateResponse);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('batch input calls updatePlatformJobs', async () => {
|
|
||||||
const { service, inputsService, updatePlatformJobsSpy } = buildService(batchOldInput);
|
|
||||||
|
|
||||||
await service.updatePipelineInput(
|
|
||||||
'pipeline-id',
|
|
||||||
'input-id',
|
|
||||||
updateInputDTO,
|
|
||||||
info,
|
|
||||||
user,
|
|
||||||
metadata,
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(updatePlatformJobsSpy).toHaveBeenCalled();
|
|
||||||
expect(inputsService.update).toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('PipelinesService - create', () => {
|
|
||||||
afterEach(() => jest.clearAllMocks());
|
|
||||||
|
|
||||||
// The body's `config` (carrying CDC destinations) must reach pi-factory as a
|
|
||||||
// JSON string — the gRPC proto field is a string, so an object would be
|
|
||||||
// stripped on the wire. Mirrors how `properties` is serialized.
|
|
||||||
it('serializes the body config into the gRPC create request', async () => {
|
|
||||||
const { service } = buildService(cdcOldInput);
|
|
||||||
|
|
||||||
let captured: any;
|
|
||||||
(service as any).pipelineWriteService = {
|
|
||||||
PipelineV2Create: (req: any) => {
|
|
||||||
captured = req;
|
|
||||||
// The service does `lastValueFrom(...)`; return a real Observable.
|
|
||||||
return of({ pipeline: {} });
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
const body: any = {
|
|
||||||
name: 'p',
|
|
||||||
input_id: 'i',
|
|
||||||
transformations_ids: [],
|
|
||||||
tags: [],
|
|
||||||
properties: { schema: 'cadastros' },
|
|
||||||
config: {
|
|
||||||
cron: '@once',
|
|
||||||
tables: [
|
|
||||||
{
|
|
||||||
name: 'pedidos',
|
|
||||||
destinations: {
|
|
||||||
raw: { table_schema: 'PUBLIC', table_name: 'pedidos_001' },
|
|
||||||
},
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
await service.create(body, metadata);
|
|
||||||
|
|
||||||
expect(typeof captured.config).toBe('string');
|
|
||||||
expect(JSON.parse(captured.config).tables[0].destinations.raw.table_name).toBe(
|
|
||||||
'pedidos_001',
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,4 +1,3 @@
|
|||||||
import { isCdcPlugin } from 'src/utils/cdc';
|
|
||||||
/* eslint-disable no-async-promise-executor */
|
/* eslint-disable no-async-promise-executor */
|
||||||
import {
|
import {
|
||||||
BadRequestException,
|
BadRequestException,
|
||||||
@@ -20,7 +19,7 @@ import { lastValueFrom } from 'rxjs';
|
|||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
import { PipelinesClientConfiguration } from './pipelines-client';
|
import { PipelinesClientConfiguration } from './pipelines-client';
|
||||||
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
import { ICreatePipelineV2Req, IIdRequest, UpdatePlatformInputRequest, UpdateTableDTO } from './interfaces';
|
||||||
import { PipelineV2CreateRequest, AddCdcJobsRequest, AddCdcJobsResponse } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/PipelineV2/interfaces/messages';
|
||||||
import { Metadata } from '@grpc/grpc-js';
|
import { Metadata } from '@grpc/grpc-js';
|
||||||
import { ConnectorClientService } from '../connector/client.service';
|
import { ConnectorClientService } from '../connector/client.service';
|
||||||
import { InputsService } from '../inputs/inputs.service';
|
import { InputsService } from '../inputs/inputs.service';
|
||||||
@@ -106,10 +105,6 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
transformations_ids: body.transformations_ids,
|
transformations_ids: body.transformations_ids,
|
||||||
tags: body.tags,
|
tags: body.tags,
|
||||||
properties: body.properties && JSON.stringify(body.properties),
|
properties: body.properties && JSON.stringify(body.properties),
|
||||||
// JSON-serialize the create body config so it survives the gRPC wire
|
|
||||||
// (the proto field is a string). pi-factory's CDC path reads
|
|
||||||
// config.tables[].destinations to honor a user-supplied raw table name.
|
|
||||||
config: body.config && JSON.stringify(body.config),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const createPipelineResponse = await lastValueFrom(
|
const createPipelineResponse = await lastValueFrom(
|
||||||
@@ -165,62 +160,6 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
return findOnePipelineResponse;
|
return findOnePipelineResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
// CDC lifecycle operations (Kafka Connect backed).
|
|
||||||
async pause(
|
|
||||||
id: string,
|
|
||||||
metadata,
|
|
||||||
): Promise<Messages.PipelineV2OperationResponse> {
|
|
||||||
this.logger.info('PipelinesClientService - Pause');
|
|
||||||
return lastValueFrom(
|
|
||||||
this.pipelineWriteService.PipelineV2Pause({ id }, metadata),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async unpause(
|
|
||||||
id: string,
|
|
||||||
metadata,
|
|
||||||
): Promise<Messages.PipelineV2OperationResponse> {
|
|
||||||
this.logger.info('PipelinesClientService - Unpause');
|
|
||||||
return lastValueFrom(
|
|
||||||
this.pipelineWriteService.PipelineV2Unpause({ id }, metadata),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async restart(
|
|
||||||
id: string,
|
|
||||||
metadata,
|
|
||||||
): Promise<Messages.PipelineV2OperationResponse> {
|
|
||||||
this.logger.info('PipelinesClientService - Restart');
|
|
||||||
return lastValueFrom(
|
|
||||||
this.pipelineWriteService.PipelineV2Restart({ id }, metadata),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async resetJobState(
|
|
||||||
id: string,
|
|
||||||
job_id: string,
|
|
||||||
schedule_minutes: number | undefined,
|
|
||||||
metadata,
|
|
||||||
): Promise<Messages.PipelineV2OperationResponse> {
|
|
||||||
this.logger.info('PipelinesClientService - ResetJobState');
|
|
||||||
return lastValueFrom(
|
|
||||||
this.pipelineWriteService.PipelineV2ResetJobState(
|
|
||||||
{ id, job_id, schedule_minutes },
|
|
||||||
metadata,
|
|
||||||
),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async getLiveStatus(
|
|
||||||
id: string,
|
|
||||||
metadata,
|
|
||||||
): Promise<Messages.PipelineV2GetLiveStatusResponse> {
|
|
||||||
this.logger.info('PipelinesClientService - GetLiveStatus');
|
|
||||||
return lastValueFrom(
|
|
||||||
this.pipelineReadService.PipelineV2GetLiveStatus({ id }, metadata),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async update(
|
async update(
|
||||||
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
|
UpdatePipelineRequest: Messages.PipelineV2UpdateRequest,
|
||||||
metadata,
|
metadata,
|
||||||
@@ -438,7 +377,6 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
});
|
});
|
||||||
|
|
||||||
this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput));
|
this.logger.info('Update Dynamo Reference :' + JSON.stringify(oldInput));
|
||||||
const isCdc = isCdcPlugin(oldInput.plugin);
|
|
||||||
const pipelineIdFormat = pipelineId.split('-').join('_');
|
const pipelineIdFormat = pipelineId.split('-').join('_');
|
||||||
const rollback: RollbackPromise[] = [];
|
const rollback: RollbackPromise[] = [];
|
||||||
|
|
||||||
@@ -499,21 +437,17 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!isCdc) {
|
try {
|
||||||
try {
|
await this.updatePlatformJobs(
|
||||||
await this.updatePlatformJobs(
|
pipelineIdFormat,
|
||||||
pipelineIdFormat,
|
updateInputResponse.input.type,
|
||||||
updateInputResponse.input.type,
|
updateInputDTO,
|
||||||
updateInputDTO,
|
user
|
||||||
user
|
);
|
||||||
);
|
} catch (error) {
|
||||||
} catch (error) {
|
this.logger.error(error);
|
||||||
this.logger.error(error);
|
await this.executeRenameRollback(rollback)
|
||||||
await this.executeRenameRollback(rollback)
|
throw new Error("Error Platform API updating jobs");
|
||||||
throw new Error("Error Platform API updating jobs");
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this.logger.info('CDC input: skipping updatePlatformJobs (batch sync_mode/memory do not apply to CDC jobs)');
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return updateInputResponse;
|
return updateInputResponse;
|
||||||
@@ -728,10 +662,6 @@ export class PipelinesService implements OnModuleInit {
|
|||||||
return statusPipelineResponse;
|
return statusPipelineResponse;
|
||||||
}
|
}
|
||||||
|
|
||||||
async addCdcJobs(data: AddCdcJobsRequest): Promise<AddCdcJobsResponse> {
|
|
||||||
return lastValueFrom(this.pipelineWriteService.AddCdcJobs(data));
|
|
||||||
}
|
|
||||||
|
|
||||||
async runPipeline({ id, info }: IIdRequest) {
|
async runPipeline({ id, info }: IIdRequest) {
|
||||||
this.logger.info('PipelinesClientService - RunPipeline');
|
this.logger.info('PipelinesClientService - RunPipeline');
|
||||||
const statusPipelineResponse = await lastValueFrom(
|
const statusPipelineResponse = await lastValueFrom(
|
||||||
|
|||||||
@@ -1,243 +0,0 @@
|
|||||||
// These service modules pull in gRPC client-config modules that read
|
|
||||||
// process.env at load time; mock them (hoisted before imports) so the spec
|
|
||||||
// needs no runtime env.
|
|
||||||
jest.mock('../inputs/inputs.service', () => ({ InputsService: class {} }));
|
|
||||||
jest.mock('../pipelinesV2/pipelines.service', () => ({ PipelinesService: class {} }));
|
|
||||||
|
|
||||||
import { NotFoundException } from '@nestjs/common';
|
|
||||||
import { PipelineTablesService } from './pipeline-tables.service';
|
|
||||||
|
|
||||||
const logger = { info: jest.fn(), error: jest.fn() };
|
|
||||||
const user = { customer_id: 'c1', customer_name: 'cust', user_id: 'u1' } as never;
|
|
||||||
const info = { customer_id: 'c1', customer: 'cust', user_id: 'u1' };
|
|
||||||
|
|
||||||
type Mocks = {
|
|
||||||
proxy: jest.Mock;
|
|
||||||
markTableDeleted: jest.Mock;
|
|
||||||
unmarkTableDeleted: jest.Mock;
|
|
||||||
addCdcTable: jest.Mock;
|
|
||||||
removeCdcTable: jest.Mock;
|
|
||||||
addCdcJobs: jest.Mock;
|
|
||||||
};
|
|
||||||
|
|
||||||
function build(): { service: PipelineTablesService; m: Mocks } {
|
|
||||||
const m: Mocks = {
|
|
||||||
proxy: jest.fn(),
|
|
||||||
markTableDeleted: jest.fn().mockResolvedValue({ is_deleted: true, deleted_at: 't' }),
|
|
||||||
unmarkTableDeleted: jest.fn().mockResolvedValue({}),
|
|
||||||
addCdcTable: jest.fn().mockResolvedValue({ input: {} }),
|
|
||||||
removeCdcTable: jest.fn().mockResolvedValue({ input: {} }),
|
|
||||||
addCdcJobs: jest.fn(),
|
|
||||||
};
|
|
||||||
type Deps = ConstructorParameters<typeof PipelineTablesService>;
|
|
||||||
const service = new PipelineTablesService(
|
|
||||||
{ proxy: m.proxy } as unknown as Deps[0],
|
|
||||||
{
|
|
||||||
markTableDeleted: m.markTableDeleted,
|
|
||||||
unmarkTableDeleted: m.unmarkTableDeleted,
|
|
||||||
addCdcTable: m.addCdcTable,
|
|
||||||
removeCdcTable: m.removeCdcTable,
|
|
||||||
} as unknown as Deps[1],
|
|
||||||
{ addCdcJobs: m.addCdcJobs } as unknown as Deps[2],
|
|
||||||
{ logger } as unknown as Deps[3],
|
|
||||||
);
|
|
||||||
return { service, m };
|
|
||||||
}
|
|
||||||
|
|
||||||
const pipelineWithJobs = (connector: string) => ({
|
|
||||||
jobs: [
|
|
||||||
{ job_id: 'p_0', input: { connector, table_name: 'pedidos' } },
|
|
||||||
{ job_id: 'p_1', input: { connector, table_name: 'clientes' } },
|
|
||||||
{ job_id: 'p_2', input: { connector, table_name: 'produtos' } },
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
const deleteCalls = (proxy: jest.Mock) =>
|
|
||||||
proxy.mock.calls.filter(([method]) => method === 'DELETE');
|
|
||||||
|
|
||||||
describe('PipelineTablesService.removeTable', () => {
|
|
||||||
afterEach(() => jest.clearAllMocks());
|
|
||||||
|
|
||||||
it.each(['cdc', 'jdbc'])(
|
|
||||||
'removes a %s table through the single pipeline jobs route',
|
|
||||||
async (connector) => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.proxy.mockImplementation((method: string) =>
|
|
||||||
Promise.resolve(method === 'GET' ? pipelineWithJobs(connector) : {}),
|
|
||||||
);
|
|
||||||
|
|
||||||
const result = await service.removeTable('pi-d', 'iid', 'pedidos', user);
|
|
||||||
|
|
||||||
expect(m.markTableDeleted).toHaveBeenCalledWith({ input_id: 'iid', table_name: 'pedidos', info });
|
|
||||||
expect(m.proxy).toHaveBeenCalledWith('GET', '/pipeline/pi_d', user);
|
|
||||||
expect(deleteCalls(m.proxy)).toEqual([
|
|
||||||
['DELETE', '/pipeline/pi_d/jobs', user, { job_ids: ['p_0'], delete_snowflake_tables: false }],
|
|
||||||
]);
|
|
||||||
expect(result).toEqual({ name: 'pedidos', is_deleted: true, deleted_at: 't' });
|
|
||||||
expect(m.unmarkTableDeleted).not.toHaveBeenCalled();
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
it('rolls back the mark when the platform delete fails', async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.proxy.mockImplementation((method: string) =>
|
|
||||||
method === 'GET' ? Promise.resolve(pipelineWithJobs('cdc')) : Promise.reject(new Error('platform boom')),
|
|
||||||
);
|
|
||||||
|
|
||||||
await expect(service.removeTable('pid', 'iid', 'pedidos', user)).rejects.toThrow('platform boom');
|
|
||||||
expect(m.unmarkTableDeleted).toHaveBeenCalledWith({ input_id: 'iid', table_name: 'pedidos', info });
|
|
||||||
});
|
|
||||||
|
|
||||||
it('404s (and rolls back) when the table has no job', async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.proxy.mockResolvedValue(pipelineWithJobs('cdc'));
|
|
||||||
|
|
||||||
await expect(service.removeTable('pid', 'iid', 'ghost', user)).rejects.toBeInstanceOf(NotFoundException);
|
|
||||||
expect(deleteCalls(m.proxy)).toHaveLength(0);
|
|
||||||
expect(m.unmarkTableDeleted).toHaveBeenCalledTimes(1);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('PipelineTablesService.removeTables', () => {
|
|
||||||
afterEach(() => jest.clearAllMocks());
|
|
||||||
|
|
||||||
it('removes N tables in ONE platform call (connectors reconfigured once)', async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.proxy.mockImplementation((method: string) =>
|
|
||||||
Promise.resolve(method === 'GET' ? pipelineWithJobs('cdc') : {}),
|
|
||||||
);
|
|
||||||
|
|
||||||
const result = await service.removeTables('pid', 'iid', ['pedidos', 'produtos'], user);
|
|
||||||
|
|
||||||
expect(m.markTableDeleted).toHaveBeenCalledTimes(2);
|
|
||||||
expect(deleteCalls(m.proxy)).toEqual([
|
|
||||||
['DELETE', '/pipeline/pid/jobs', user, { job_ids: ['p_0', 'p_2'], delete_snowflake_tables: false }],
|
|
||||||
]);
|
|
||||||
expect(result).toEqual({ table_names: ['pedidos', 'produtos'], deleted: true });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("rolls back only this call's marks when the platform delete fails", async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.proxy.mockImplementation((method: string) =>
|
|
||||||
method === 'GET' ? Promise.resolve(pipelineWithJobs('cdc')) : Promise.reject(new Error('platform boom')),
|
|
||||||
);
|
|
||||||
|
|
||||||
await expect(service.removeTables('pid', 'iid', ['pedidos', 'clientes'], user)).rejects.toThrow();
|
|
||||||
|
|
||||||
const unmarked = m.unmarkTableDeleted.mock.calls.map(([arg]) => arg.table_name).sort();
|
|
||||||
expect(unmarked).toEqual(['clientes', 'pedidos']);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rolls back the marks made so far if a later mark fails (atomic)', async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.markTableDeleted
|
|
||||||
.mockResolvedValueOnce({ is_deleted: true, deleted_at: 't' })
|
|
||||||
.mockRejectedValueOnce(new Error('dynamo boom'));
|
|
||||||
|
|
||||||
await expect(service.removeTables('pid', 'iid', ['pedidos', 'clientes'], user)).rejects.toThrow('dynamo boom');
|
|
||||||
|
|
||||||
expect(m.unmarkTableDeleted).toHaveBeenCalledTimes(1);
|
|
||||||
expect(m.unmarkTableDeleted.mock.calls[0][0].table_name).toBe('pedidos');
|
|
||||||
expect(m.proxy).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('404s when a requested table has no matching job and rolls back the marks', async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.proxy.mockResolvedValue(pipelineWithJobs('cdc'));
|
|
||||||
|
|
||||||
await expect(service.removeTables('pid', 'iid', ['pedidos', 'ghost'], user)).rejects.toBeInstanceOf(NotFoundException);
|
|
||||||
expect(m.unmarkTableDeleted).toHaveBeenCalledTimes(2);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('keeps going when a rollback step itself fails', async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.proxy.mockImplementation((method: string) =>
|
|
||||||
method === 'GET' ? Promise.resolve(pipelineWithJobs('cdc')) : Promise.reject(new Error('platform boom')),
|
|
||||||
);
|
|
||||||
m.unmarkTableDeleted.mockRejectedValueOnce(new Error('unmark boom'));
|
|
||||||
|
|
||||||
await expect(service.removeTables('pid', 'iid', ['pedidos', 'clientes'], user)).rejects.toThrow('platform boom');
|
|
||||||
expect(m.unmarkTableDeleted).toHaveBeenCalledTimes(2);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe('PipelineTablesService.addTable', () => {
|
|
||||||
afterEach(() => jest.clearAllMocks());
|
|
||||||
|
|
||||||
const body = {
|
|
||||||
table_name: 'orders',
|
|
||||||
table_schema: 'public',
|
|
||||||
primary_keys: ['id'],
|
|
||||||
destinations: {
|
|
||||||
raw: { table_schema: 'raw', table_name: 'orders' },
|
|
||||||
qualify: { table_schema: 'qualify', table_name: 'orders' },
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
const storedTable = (extra: Record<string, unknown> = {}) => ({
|
|
||||||
table_schema: 'public',
|
|
||||||
table_name: 'orders',
|
|
||||||
name: 'orders',
|
|
||||||
primary_keys: ['id'],
|
|
||||||
iceberg_table_name: undefined,
|
|
||||||
iceberg_qualify_table_name: undefined,
|
|
||||||
columns: [],
|
|
||||||
column_exclude_list: [],
|
|
||||||
...extra,
|
|
||||||
});
|
|
||||||
|
|
||||||
it('appends to DynamoDB, then dispatches the platform jobs', async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.addCdcJobs.mockResolvedValue({ job_ids: ['p_2'], skipped: [] });
|
|
||||||
|
|
||||||
const result = await service.addTable('pid', 'iid', body, user);
|
|
||||||
|
|
||||||
expect(m.addCdcTable).toHaveBeenCalledWith({ client_id: 'c1', id: 'iid', table: storedTable(), info });
|
|
||||||
expect(m.addCdcJobs).toHaveBeenCalledWith({
|
|
||||||
pipeline_id: 'pid',
|
|
||||||
input_id: 'iid',
|
|
||||||
tables: [{ table_schema: 'public', table_name: 'orders', primary_keys: ['id'], destinations: body.destinations }],
|
|
||||||
info,
|
|
||||||
});
|
|
||||||
expect(result).toEqual({ job_ids: ['p_2'], skipped: [] });
|
|
||||||
expect(m.removeCdcTable).not.toHaveBeenCalled();
|
|
||||||
expect(m.addCdcTable.mock.invocationCallOrder[0]).toBeLessThan(m.addCdcJobs.mock.invocationCallOrder[0]);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('carries the iceberg names and columns through to the stored CdcTable', async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.addCdcJobs.mockResolvedValue({ job_ids: ['p_3'], skipped: [] });
|
|
||||||
const columns = [
|
|
||||||
{ name: 'id', type: 'int', is_primary_key: true },
|
|
||||||
{ name: 'descr', type: 'varchar(255)', is_primary_key: false },
|
|
||||||
];
|
|
||||||
|
|
||||||
await service.addTable('pid', 'iid', {
|
|
||||||
...body,
|
|
||||||
iceberg_table_name: 'cdc_raw.public__orders',
|
|
||||||
iceberg_qualify_table_name: 'orders_dedup',
|
|
||||||
columns,
|
|
||||||
column_exclude_list: ['descr'],
|
|
||||||
}, user);
|
|
||||||
|
|
||||||
expect(m.addCdcTable).toHaveBeenCalledWith({
|
|
||||||
client_id: 'c1',
|
|
||||||
id: 'iid',
|
|
||||||
table: storedTable({
|
|
||||||
iceberg_table_name: 'cdc_raw.public__orders',
|
|
||||||
iceberg_qualify_table_name: 'orders_dedup',
|
|
||||||
columns,
|
|
||||||
column_exclude_list: ['descr'],
|
|
||||||
}),
|
|
||||||
info,
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
it('rolls back the DynamoDB row when AddJobs fails', async () => {
|
|
||||||
const { service, m } = build();
|
|
||||||
m.addCdcJobs.mockRejectedValue(new Error('grpc boom'));
|
|
||||||
|
|
||||||
await expect(service.addTable('pid', 'iid', body, user)).rejects.toThrow('grpc boom');
|
|
||||||
expect(m.removeCdcTable).toHaveBeenCalledWith({ client_id: 'c1', id: 'iid', table_name: 'orders', info });
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,157 +0,0 @@
|
|||||||
import { Inject, Injectable, NotFoundException } from '@nestjs/common';
|
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|
||||||
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
|
|
||||||
|
|
||||||
import { RequestUser } from '../../decorators/user.decorator';
|
|
||||||
import { InputsService } from '../inputs/inputs.service';
|
|
||||||
import { toCdcTable } from '../inputs/cdc-table.mapper';
|
|
||||||
import { PipelinesService } from '../pipelinesV2/pipelines.service';
|
|
||||||
import { PlatformApiService } from './platform-api.service';
|
|
||||||
import { AddCdcTableBody } from './platform-api.dto';
|
|
||||||
|
|
||||||
interface PlatformJob {
|
|
||||||
job_id: string;
|
|
||||||
input?: { table_name?: string; connector?: string } | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface PlatformPipeline {
|
|
||||||
jobs?: PlatformJob[] | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Add/remove tables of a pipeline: the DynamoDB input (soft-delete flags or
|
|
||||||
* a new CdcTable row) and the platform-api jobs move together, with the
|
|
||||||
* DynamoDB side rolled back when the platform side fails.
|
|
||||||
*
|
|
||||||
* Removal always goes through DELETE /pipeline/{id}/jobs — the one platform
|
|
||||||
* route that dispatches by pipeline type (Airflow refresh for batch, Kafka
|
|
||||||
* Connect reconfiguration for CDC). Landed destination data is kept
|
|
||||||
* (delete_snowflake_tables: false), matching the soft-delete in DynamoDB.
|
|
||||||
*/
|
|
||||||
@Injectable()
|
|
||||||
export class PipelineTablesService {
|
|
||||||
logger: DadosferaLogger;
|
|
||||||
|
|
||||||
constructor(
|
|
||||||
private readonly platformApiService: PlatformApiService,
|
|
||||||
private readonly inputsService: InputsService,
|
|
||||||
private readonly pipelinesClientService: PipelinesService,
|
|
||||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
|
||||||
) {
|
|
||||||
this.logger = dadosferaLogger.logger;
|
|
||||||
}
|
|
||||||
|
|
||||||
async removeTable(pipelineId: string, inputId: string, tableName: string, user: RequestUser) {
|
|
||||||
const info = this.infoOf(user);
|
|
||||||
|
|
||||||
this.logger.info('removeTable: marking table as deleted', { inputId, tableName });
|
|
||||||
const marked = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info });
|
|
||||||
|
|
||||||
try {
|
|
||||||
const jobIds = await this.resolveJobIds(pipelineId, [tableName], user);
|
|
||||||
await this.removeJobs(pipelineId, jobIds, user);
|
|
||||||
return { name: tableName, is_deleted: marked.is_deleted ?? true, deleted_at: marked.deleted_at };
|
|
||||||
} catch (error) {
|
|
||||||
this.logger.error('removeTable: platform-api delete failed, rolling back the mark', { tableName, error: error.message });
|
|
||||||
await this.unmarkAll(inputId, [tableName], info);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Removes N tables with ONE platform call, so CDC connectors are reconfigured once. */
|
|
||||||
async removeTables(pipelineId: string, inputId: string, tableNames: string[], user: RequestUser) {
|
|
||||||
const info = this.infoOf(user);
|
|
||||||
|
|
||||||
// Soft-delete each table, tracking which succeeded so a later failure
|
|
||||||
// only rolls back the marks made in THIS call.
|
|
||||||
const marked: string[] = [];
|
|
||||||
try {
|
|
||||||
for (const name of tableNames) {
|
|
||||||
await this.inputsService.markTableDeleted({ input_id: inputId, table_name: name, info });
|
|
||||||
marked.push(name);
|
|
||||||
}
|
|
||||||
|
|
||||||
const jobIds = await this.resolveJobIds(pipelineId, tableNames, user);
|
|
||||||
await this.removeJobs(pipelineId, jobIds, user);
|
|
||||||
return { table_names: tableNames, deleted: true };
|
|
||||||
} catch (error) {
|
|
||||||
this.logger.error("removeTables: failed, rolling back this call's marks", { tableNames, error: error.message });
|
|
||||||
await this.unmarkAll(inputId, marked, info);
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Appends a CDC table to the DynamoDB input, then dispatches its platform jobs. */
|
|
||||||
async addTable(pipelineId: string, inputId: string, body: AddCdcTableBody, user: RequestUser) {
|
|
||||||
const info = this.infoOf(user);
|
|
||||||
const cdcTable = toCdcTable(body);
|
|
||||||
|
|
||||||
this.logger.info('addTable: appending CDC table to DynamoDB', { inputId, tableName: body.table_name });
|
|
||||||
await this.inputsService.addCdcTable({ client_id: user.customer_id, id: inputId, table: cdcTable, info });
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await this.pipelinesClientService.addCdcJobs({
|
|
||||||
pipeline_id: pipelineId,
|
|
||||||
input_id: inputId,
|
|
||||||
tables: [{
|
|
||||||
table_schema: body.table_schema,
|
|
||||||
table_name: body.table_name,
|
|
||||||
primary_keys: body.primary_keys,
|
|
||||||
destinations: body.destinations,
|
|
||||||
}],
|
|
||||||
info,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
this.logger.error('addTable: platform AddJobs failed, rolling back the DynamoDB row', { tableName: body.table_name, error: error.message });
|
|
||||||
try {
|
|
||||||
await this.inputsService.removeCdcTable({ client_id: user.customer_id, id: inputId, table_name: body.table_name, info });
|
|
||||||
} catch (rollbackError) {
|
|
||||||
this.logger.error('addTable: rollback failed', { error: rollbackError.message });
|
|
||||||
}
|
|
||||||
throw error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private infoOf(user: RequestUser): Info {
|
|
||||||
return { customer_id: user.customer_id, customer: user.customer_name, user_id: user.user_id };
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Platform-API replaces '-' with '_' in pipeline IDs. */
|
|
||||||
private normalizePipelineId(id: string): string {
|
|
||||||
return id.replace(/-/g, '_');
|
|
||||||
}
|
|
||||||
|
|
||||||
private async resolveJobIds(pipelineId: string, tableNames: string[], user: RequestUser): Promise<string[]> {
|
|
||||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
|
||||||
const pipeline: PlatformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
|
|
||||||
const jobs = pipeline?.jobs ?? [];
|
|
||||||
|
|
||||||
return tableNames.map((name) => {
|
|
||||||
const job = jobs.find((j) => j.input?.table_name === name);
|
|
||||||
if (!job) throw new NotFoundException(`Job for table '${name}' not found in pipeline`);
|
|
||||||
return job.job_id;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
private async removeJobs(pipelineId: string, jobIds: string[], user: RequestUser) {
|
|
||||||
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
|
||||||
this.logger.info('removeJobs: deleting jobs from platform-api', { jobIds });
|
|
||||||
await this.platformApiService.proxy(
|
|
||||||
'DELETE',
|
|
||||||
`/pipeline/${normalizedPipelineId}/jobs`,
|
|
||||||
user,
|
|
||||||
{ job_ids: jobIds, delete_snowflake_tables: false },
|
|
||||||
);
|
|
||||||
this.logger.info('removeJobs: jobs deleted', { jobIds });
|
|
||||||
}
|
|
||||||
|
|
||||||
private async unmarkAll(inputId: string, tableNames: string[], info: Info) {
|
|
||||||
for (const name of tableNames) {
|
|
||||||
try {
|
|
||||||
await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: name, info });
|
|
||||||
} catch (rollbackError) {
|
|
||||||
this.logger.error('rollback failed: table stays marked as deleted', { tableName: name, error: rollbackError.message });
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
// These service modules pull in gRPC client-config modules that read
|
|
||||||
// process.env at load time; mock them (hoisted before imports) so the spec
|
|
||||||
// needs no runtime env. Each mock severs an entire import subtree and still
|
|
||||||
// provides a class usable as a DI token.
|
|
||||||
jest.mock('../customers/customers.service', () => ({ CustomersService: class {} }));
|
|
||||||
jest.mock('../catalog/catalog.service', () => ({ CatalogService: class {} }));
|
|
||||||
jest.mock('../inputs/inputs.service', () => ({ InputsService: class {} }));
|
|
||||||
jest.mock('./pipeline-tables.service', () => ({ PipelineTablesService: class {} }));
|
|
||||||
|
|
||||||
import { BadRequestException } from '@nestjs/common';
|
|
||||||
import { Test, TestingModule } from '@nestjs/testing';
|
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|
||||||
|
|
||||||
import { PlatformApiController } from './platform-api.controller';
|
|
||||||
import { PlatformApiService } from './platform-api.service';
|
|
||||||
import { PipelineTablesService } from './pipeline-tables.service';
|
|
||||||
import { ElasticsearchService } from '../../services/elasticsearch';
|
|
||||||
import { DynamoDBService } from '../../services/dynamodb';
|
|
||||||
import { CustomersService } from '../customers/customers.service';
|
|
||||||
import { CatalogService } from '../catalog/catalog.service';
|
|
||||||
import { InputsService } from '../inputs/inputs.service';
|
|
||||||
|
|
||||||
const logger = { info: jest.fn(), error: jest.fn() };
|
|
||||||
const mockUser = { customer_id: 'c1', customer_name: 'cust', user_id: 'u1' } as never;
|
|
||||||
|
|
||||||
// The table routes are thin: validate the body, delegate to
|
|
||||||
// PipelineTablesService (covered in pipeline-tables.service.spec.ts).
|
|
||||||
describe('PlatformApiController - table routes', () => {
|
|
||||||
let controller: PlatformApiController;
|
|
||||||
let tables: { removeTable: jest.Mock; removeTables: jest.Mock; addTable: jest.Mock };
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
tables = { removeTable: jest.fn(), removeTables: jest.fn(), addTable: jest.fn() };
|
|
||||||
|
|
||||||
const module: TestingModule = await Test.createTestingModule({
|
|
||||||
controllers: [PlatformApiController],
|
|
||||||
providers: [
|
|
||||||
{ provide: PlatformApiService, useValue: {} },
|
|
||||||
{ provide: ElasticsearchService, useValue: {} },
|
|
||||||
{ provide: DynamoDBService, useValue: {} },
|
|
||||||
{ provide: CustomersService, useValue: {} },
|
|
||||||
{ provide: CatalogService, useValue: {} },
|
|
||||||
{ provide: InputsService, useValue: {} },
|
|
||||||
{ provide: PipelineTablesService, useValue: tables },
|
|
||||||
{ provide: DadosferaLogger, useValue: { logger } },
|
|
||||||
],
|
|
||||||
}).compile();
|
|
||||||
|
|
||||||
controller = module.get<PlatformApiController>(PlatformApiController);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('deleteTable delegates', async () => {
|
|
||||||
tables.removeTable.mockResolvedValue({ name: 'pedidos', is_deleted: true });
|
|
||||||
await expect(controller.deleteTable('pid', 'iid', { table_name: 'pedidos' }, mockUser))
|
|
||||||
.resolves.toEqual({ name: 'pedidos', is_deleted: true });
|
|
||||||
expect(tables.removeTable).toHaveBeenCalledWith('pid', 'iid', 'pedidos', mockUser);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('deleteTables delegates', async () => {
|
|
||||||
tables.removeTables.mockResolvedValue({ deleted: true });
|
|
||||||
await controller.deleteTables('pid', 'iid', { table_names: ['a', 'b'] }, mockUser);
|
|
||||||
expect(tables.removeTables).toHaveBeenCalledWith('pid', 'iid', ['a', 'b'], mockUser);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('deleteTables rejects an empty table_names before touching anything', async () => {
|
|
||||||
await expect(controller.deleteTables('pid', 'iid', { table_names: [] }, mockUser))
|
|
||||||
.rejects.toBeInstanceOf(BadRequestException);
|
|
||||||
expect(tables.removeTables).not.toHaveBeenCalled();
|
|
||||||
});
|
|
||||||
|
|
||||||
it('addTable delegates', async () => {
|
|
||||||
const body = {
|
|
||||||
table_name: 'orders',
|
|
||||||
table_schema: 'public',
|
|
||||||
primary_keys: ['id'],
|
|
||||||
destinations: {
|
|
||||||
raw: { table_schema: 'raw', table_name: 'orders' },
|
|
||||||
qualify: { table_schema: 'qualify', table_name: 'orders' },
|
|
||||||
},
|
|
||||||
};
|
|
||||||
tables.addTable.mockResolvedValue({ job_ids: ['p_2'], skipped: [] });
|
|
||||||
await expect(controller.addTable('pid', 'iid', body, mockUser)).resolves.toEqual({ job_ids: ['p_2'], skipped: [] });
|
|
||||||
expect(tables.addTable).toHaveBeenCalledWith('pid', 'iid', body, mockUser);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -11,6 +11,7 @@ import {
|
|||||||
Inject,
|
Inject,
|
||||||
BadRequestException,
|
BadRequestException,
|
||||||
HttpException,
|
HttpException,
|
||||||
|
NotFoundException,
|
||||||
UseGuards,
|
UseGuards,
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
|
import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger';
|
||||||
@@ -30,9 +31,8 @@ import { CustomersService } from '../customers/customers.service';
|
|||||||
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
|
import { validateCronAgainstScheduleLimit } from '../../utils/cron-validation';
|
||||||
import { CatalogService } from '../catalog/catalog.service';
|
import { CatalogService } from '../catalog/catalog.service';
|
||||||
import { PackTheMetadata } from '../../utils/PackTheMetadata';
|
import { PackTheMetadata } from '../../utils/PackTheMetadata';
|
||||||
import { AddCdcTableBody, DeleteTableBody, DeleteTablesBody, ValidationTableDTO } from './platform-api.dto';
|
import { ValidationTableDTO } from './platform-api.dto';
|
||||||
import { InputsService } from '../inputs/inputs.service';
|
import { InputsService } from '../inputs/inputs.service';
|
||||||
import { PipelineTablesService } from './pipeline-tables.service';
|
|
||||||
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
|
import { PipelineExecutionGuard } from 'src/guards/pipeline-execution.guard';
|
||||||
|
|
||||||
|
|
||||||
@@ -61,7 +61,6 @@ export class PlatformApiController {
|
|||||||
private readonly customersService: CustomersService,
|
private readonly customersService: CustomersService,
|
||||||
private readonly catalogService: CatalogService,
|
private readonly catalogService: CatalogService,
|
||||||
private readonly inputsService: InputsService,
|
private readonly inputsService: InputsService,
|
||||||
private readonly pipelineTablesService: PipelineTablesService,
|
|
||||||
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
@Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger,
|
||||||
) {
|
) {
|
||||||
this.logger = dadosferaLogger.logger;
|
this.logger = dadosferaLogger.logger;
|
||||||
@@ -546,20 +545,6 @@ export class PlatformApiController {
|
|||||||
return this.platformApiService.proxy('GET', `/pipeline/${normalizedId}`, user);
|
return this.platformApiService.proxy('GET', `/pipeline/${normalizedId}`, user);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Get('iceberg/namespaces')
|
|
||||||
@ApiOperation({ summary: 'List existing Polaris Iceberg namespaces (CDC destination dropdown)' })
|
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
|
||||||
async getIcebergNamespaces(@User() user: RequestUser) {
|
|
||||||
return this.platformApiService.proxy('GET', '/iceberg/namespaces', user);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post('iceberg/tables/validate')
|
|
||||||
@ApiOperation({ summary: 'Validate CDC Iceberg raw table names against Polaris' })
|
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
|
|
||||||
async validateIcebergTables(@Body() body: any, @User() user: RequestUser) {
|
|
||||||
return this.platformApiService.proxy('POST', '/iceberg/tables/validate', user, body);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Patch('pipelines/:pipelineId')
|
@Patch('pipelines/:pipelineId')
|
||||||
@ApiOperation({ summary: 'Update pipeline by ID' })
|
@ApiOperation({ summary: 'Update pipeline by ID' })
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
||||||
@@ -987,46 +972,49 @@ export class PlatformApiController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Delete('pipelines/:pipelineId/inputs/:inputId')
|
@Delete('pipelines/:pipelineId/inputs/:inputId')
|
||||||
@ApiOperation({ summary: 'Mark a table as deleted and remove its job via platform-api' })
|
@ApiOperation({ summary: 'Mark a table as deleted and delete its associated job via platform-api' })
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
||||||
@UseGuards(PipelineExecutionGuard)
|
@UseGuards(PipelineExecutionGuard)
|
||||||
async deleteTable(
|
async deleteTable(
|
||||||
@Param('pipelineId') pipelineId: string,
|
@Param('pipelineId') pipelineId: string,
|
||||||
@Param('inputId') inputId: string,
|
@Param('inputId') inputId: string,
|
||||||
@Body() body: DeleteTableBody,
|
@Body() body: { table_name: string },
|
||||||
@User() user: RequestUser,
|
@User() user: RequestUser,
|
||||||
) {
|
) {
|
||||||
return this.pipelineTablesService.removeTable(pipelineId, inputId, body.table_name, user);
|
const tableName = body.table_name;
|
||||||
}
|
const info = {
|
||||||
|
customer_id: user.customer_id,
|
||||||
|
customer: user.customer_name,
|
||||||
|
user_id: user.user_id,
|
||||||
|
};
|
||||||
|
|
||||||
@Delete('pipelines/:pipelineId/inputs/:inputId/tables')
|
this.logger.info('deleteTable: marking table as deleted', { inputId, tableName });
|
||||||
@ApiOperation({ summary: 'Batch-remove tables from an input; reconfigures the CDC connectors once' })
|
const updatedInput: any = await this.inputsService.markTableDeleted({ input_id: inputId, table_name: tableName, info });
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
|
this.logger.info('deleteTable: table marked as deleted', { inputId, tableName });
|
||||||
@UseGuards(PipelineExecutionGuard)
|
|
||||||
async deleteTables(
|
try {
|
||||||
@Param('pipelineId') pipelineId: string,
|
const normalizedPipelineId = this.normalizePipelineId(pipelineId);
|
||||||
@Param('inputId') inputId: string,
|
this.logger.info('deleteTable: fetching pipeline from platform-api', { pipelineId, normalizedPipelineId });
|
||||||
@Body() body: DeleteTablesBody,
|
const platformPipeline = await this.platformApiService.proxy('GET', `/pipeline/${normalizedPipelineId}`, user);
|
||||||
@User() user: RequestUser,
|
this.logger.info('deleteTable: pipeline fetched', { jobCount: platformPipeline?.jobs?.length });
|
||||||
) {
|
|
||||||
const tableNames = body.table_names ?? [];
|
const job = platformPipeline?.jobs?.find((j: any) => j.input?.table_name === tableName);
|
||||||
if (tableNames.length === 0) {
|
if (!job) throw new NotFoundException(`Job for table '${tableName}' not found in pipeline`);
|
||||||
throw new BadRequestException('table_names must be a non-empty array');
|
|
||||||
|
this.logger.info('deleteTable: deleting job from platform-api', { jobId: job.job_id });
|
||||||
|
await this.platformApiService.proxy('DELETE', `/jobs/${job.job_id}`, user);
|
||||||
|
this.logger.info('deleteTable: job deleted', { jobId: job.job_id });
|
||||||
|
|
||||||
|
return { name: tableName, is_deleted: updatedInput.is_deleted ?? true, deleted_at: updatedInput.deleted_at };
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('deleteTable: platform-api delete failed, attempting rollback', { tableName, error: error.message });
|
||||||
|
try {
|
||||||
|
await this.inputsService.unmarkTableDeleted({ input_id: inputId, table_name: tableName, info });
|
||||||
|
} catch (rollbackError) {
|
||||||
|
this.logger.error('deleteTable: rollback failed', { tableName, error: rollbackError.message });
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
}
|
}
|
||||||
return this.pipelineTablesService.removeTables(pipelineId, inputId, tableNames, user);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Post('pipelines/:pipelineId/inputs/:inputId/tables')
|
|
||||||
@ApiOperation({ summary: 'Add a CDC table to an input and dispatch its platform jobs' })
|
|
||||||
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
|
|
||||||
@UseGuards(PipelineExecutionGuard)
|
|
||||||
async addTable(
|
|
||||||
@Param('pipelineId') pipelineId: string,
|
|
||||||
@Param('inputId') inputId: string,
|
|
||||||
@Body() body: AddCdcTableBody,
|
|
||||||
@User() user: RequestUser,
|
|
||||||
) {
|
|
||||||
return this.pipelineTablesService.addTable(pipelineId, inputId, body, user);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
|
// ==================== JOBS - JDBC SYNC MODE ROUTES ====================
|
||||||
|
|||||||
@@ -1,66 +1,9 @@
|
|||||||
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
|
import { ApiProperty } from "@nestjs/swagger";
|
||||||
|
|
||||||
export class ValidationTableDTO {
|
export class ValidationTableDTO {
|
||||||
@ApiProperty()
|
@ApiProperty()
|
||||||
tables: Array<{
|
tables: Array<{
|
||||||
table_name: string;
|
table_name: string;
|
||||||
table_schema: string;
|
table_schema: string;
|
||||||
}>;
|
}>
|
||||||
}
|
}
|
||||||
|
|
||||||
export class DeleteTableBody {
|
|
||||||
@ApiProperty()
|
|
||||||
table_name: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class DeleteTablesBody {
|
|
||||||
@ApiProperty({ type: [String] })
|
|
||||||
table_names: string[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CdcTableDestination {
|
|
||||||
@ApiProperty()
|
|
||||||
table_schema: string;
|
|
||||||
@ApiProperty()
|
|
||||||
table_name: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CdcTableDestinations {
|
|
||||||
@ApiProperty({ type: CdcTableDestination })
|
|
||||||
raw: CdcTableDestination;
|
|
||||||
@ApiProperty({ type: CdcTableDestination })
|
|
||||||
qualify: CdcTableDestination;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CdcColumnBody {
|
|
||||||
@ApiProperty()
|
|
||||||
name: string;
|
|
||||||
@ApiProperty()
|
|
||||||
type: string;
|
|
||||||
@ApiProperty()
|
|
||||||
is_primary_key: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class AddCdcTableBody {
|
|
||||||
@ApiProperty()
|
|
||||||
table_name: string;
|
|
||||||
@ApiProperty()
|
|
||||||
table_schema: string;
|
|
||||||
@ApiProperty({ type: [String] })
|
|
||||||
primary_keys: string[];
|
|
||||||
@ApiProperty({ type: CdcTableDestinations })
|
|
||||||
destinations: CdcTableDestinations;
|
|
||||||
// Iceberg destination only (protospack CdcTable.iceberg_table_name);
|
|
||||||
// absent for snowflake, back-compat.
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
iceberg_table_name?: string;
|
|
||||||
// Per-table deduped (qualify) Iceberg table name; absent => same as raw.
|
|
||||||
@ApiPropertyOptional()
|
|
||||||
iceberg_qualify_table_name?: string;
|
|
||||||
// Source column schema for iceberg deduped table pre-create.
|
|
||||||
@ApiPropertyOptional({ type: [CdcColumnBody] })
|
|
||||||
columns?: CdcColumnBody[];
|
|
||||||
// Columns the user chose to ignore -> Debezium column.exclude.list.
|
|
||||||
@ApiPropertyOptional({ type: [String] })
|
|
||||||
column_exclude_list?: string[];
|
|
||||||
}
|
|
||||||
@@ -1,28 +1,19 @@
|
|||||||
import { Module, forwardRef } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
|
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
||||||
|
|
||||||
import { PlatformApiController } from './platform-api.controller';
|
import { PlatformApiController } from './platform-api.controller';
|
||||||
import { PlatformApiService } from './platform-api.service';
|
import { PlatformApiService } from './platform-api.service';
|
||||||
import { PipelineTablesService } from './pipeline-tables.service';
|
|
||||||
import { ElasticsearchModule } from '../../services/elasticsearch';
|
import { ElasticsearchModule } from '../../services/elasticsearch';
|
||||||
import { DynamoDBModule } from '../../services/dynamodb';
|
import { DynamoDBModule } from '../../services/dynamodb';
|
||||||
import { CustomersModule } from '../customers/customers.module';
|
import { CustomersModule } from '../customers/customers.module';
|
||||||
import { CatalogModule } from '../catalog/catalog.module';
|
import { CatalogModule } from '../catalog/catalog.module';
|
||||||
import { InputsModule } from '../inputs/inputs.module';
|
import { InputsModule } from '../inputs/inputs.module';
|
||||||
import { PipelinesV2Module } from '../pipelinesV2/pipelines.module';
|
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [ElasticsearchModule, DynamoDBModule, CustomersModule, CatalogModule, InputsModule],
|
||||||
ElasticsearchModule,
|
|
||||||
DynamoDBModule,
|
|
||||||
CustomersModule,
|
|
||||||
CatalogModule,
|
|
||||||
InputsModule,
|
|
||||||
forwardRef(() => PipelinesV2Module),
|
|
||||||
],
|
|
||||||
controllers: [PlatformApiController],
|
controllers: [PlatformApiController],
|
||||||
providers: [PlatformApiService, PipelineTablesService, DadosferaLogger],
|
providers: [PlatformApiService, DadosferaLogger],
|
||||||
exports: [PlatformApiService],
|
exports: [PlatformApiService],
|
||||||
})
|
})
|
||||||
export class PlatformApiModule {}
|
export class PlatformApiModule {}
|
||||||
|
|||||||
@@ -1,26 +1,14 @@
|
|||||||
import { Test, TestingModule } from '@nestjs/testing';
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|
||||||
import { ReleaseNoteController } from './release_note.controller';
|
import { ReleaseNoteController } from './release_note.controller';
|
||||||
import { ReleaseNoteService } from './release_note.service';
|
import { ReleaseNoteService } from './release_note.service';
|
||||||
|
|
||||||
const logger = {
|
|
||||||
info: (...args) => args,
|
|
||||||
error: (...args) => args,
|
|
||||||
};
|
|
||||||
|
|
||||||
describe('ReleaseNoteController', () => {
|
describe('ReleaseNoteController', () => {
|
||||||
let controller: ReleaseNoteController;
|
let controller: ReleaseNoteController;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
const module: TestingModule = await Test.createTestingModule({
|
const module: TestingModule = await Test.createTestingModule({
|
||||||
controllers: [ReleaseNoteController],
|
controllers: [ReleaseNoteController],
|
||||||
providers: [
|
providers: [ReleaseNoteService],
|
||||||
ReleaseNoteService,
|
|
||||||
{
|
|
||||||
provide: DadosferaLogger,
|
|
||||||
useValue: { logger },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
|
controller = module.get<ReleaseNoteController>(ReleaseNoteController);
|
||||||
|
|||||||
@@ -1,24 +1,12 @@
|
|||||||
import { Test, TestingModule } from '@nestjs/testing';
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
|
|
||||||
import { ReleaseNoteService } from './release_note.service';
|
import { ReleaseNoteService } from './release_note.service';
|
||||||
|
|
||||||
const logger = {
|
|
||||||
info: (...args) => args,
|
|
||||||
error: (...args) => args,
|
|
||||||
};
|
|
||||||
|
|
||||||
describe('ReleaseNoteService', () => {
|
describe('ReleaseNoteService', () => {
|
||||||
let service: ReleaseNoteService;
|
let service: ReleaseNoteService;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
const module: TestingModule = await Test.createTestingModule({
|
const module: TestingModule = await Test.createTestingModule({
|
||||||
providers: [
|
providers: [ReleaseNoteService],
|
||||||
ReleaseNoteService,
|
|
||||||
{
|
|
||||||
provide: DadosferaLogger,
|
|
||||||
useValue: { logger },
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
service = module.get<ReleaseNoteService>(ReleaseNoteService);
|
service = module.get<ReleaseNoteService>(ReleaseNoteService);
|
||||||
|
|||||||
@@ -315,14 +315,6 @@ export function EnrichErrorCode(code: string) {
|
|||||||
'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
|
'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
|
||||||
code,
|
code,
|
||||||
};
|
};
|
||||||
case ErrorCodes.CATALOG.COLUMN_NOT_FOUND:
|
|
||||||
return {
|
|
||||||
statusCode: HttpStatus.NOT_FOUND,
|
|
||||||
error: 'Coluna não encontrada',
|
|
||||||
message:
|
|
||||||
'Uma ou mais colunas informadas não existem neste ativo. Verifique os nomes e tente novamente.',
|
|
||||||
code,
|
|
||||||
};
|
|
||||||
case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
|
case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
|
||||||
return {
|
return {
|
||||||
statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
|
statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
|
||||||
|
|||||||
@@ -1,34 +0,0 @@
|
|||||||
/**
|
|
||||||
* Single source of truth for "is this CDC?" across maestro.
|
|
||||||
*
|
|
||||||
* Two shapes carry the discriminator:
|
|
||||||
* - a connection/input `plugin` (`mysql_cdc`, ...), used before a platform
|
|
||||||
* pipeline exists (create flows, DynamoDB inputs);
|
|
||||||
* - a platform job `input.connector === 'cdc'`, the explicit type the
|
|
||||||
* platform-api stamps on every CDC job.
|
|
||||||
*
|
|
||||||
* Keep the plugin list in sync with pi-factory (`CDC_PLUGINS`) and in-factory
|
|
||||||
* (`cdcPlugins`).
|
|
||||||
*/
|
|
||||||
export const CDC_PLUGINS: readonly string[] = ['mysql_cdc', 'postgresql_cdc', 'oracle_cdc'];
|
|
||||||
|
|
||||||
export function isCdcPlugin(plugin?: string | null): boolean {
|
|
||||||
return plugin != null && CDC_PLUGINS.includes(plugin);
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PlatformJobLike {
|
|
||||||
input?: { connector?: string } | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PlatformPipelineLike {
|
|
||||||
jobs?: PlatformJobLike[] | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isCdcJob(job?: PlatformJobLike | null): boolean {
|
|
||||||
return job?.input?.connector === 'cdc';
|
|
||||||
}
|
|
||||||
|
|
||||||
/** A pipeline is CDC when any of its platform jobs is a CDC job. */
|
|
||||||
export function isCdcPipeline(pipeline?: PlatformPipelineLike | null): boolean {
|
|
||||||
return (pipeline?.jobs ?? []).some((job) => isCdcJob(job));
|
|
||||||
}
|
|
||||||
@@ -74,7 +74,6 @@ const CATALOG = {
|
|||||||
DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
|
DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
|
||||||
PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
|
PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
|
||||||
METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
|
METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
|
||||||
COLUMN_NOT_FOUND: 'CATALOG.COLUMN_NOT_FOUND',
|
|
||||||
};
|
};
|
||||||
const IDENTITY_PROVIDER = {
|
const IDENTITY_PROVIDER = {
|
||||||
INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',
|
INVALID_RESPONSE: 'IDENTITY_PROVIDER.INVALID_RESPONSE',
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
/**
|
||||||
|
* Jest setupFiles hook: provide dummy values for env vars that modules read
|
||||||
|
* at *import time* (module-load side effects), so unit specs that transitively
|
||||||
|
* import those modules don't crash before a single test runs.
|
||||||
|
*
|
||||||
|
* These are never exercised by unit tests — the services that use them are
|
||||||
|
* stubbed via DI — but the values must exist because the reads happen at
|
||||||
|
* module evaluation, before any mock is installed.
|
||||||
|
*
|
||||||
|
* DUC_URL: read at load time in src/modules/duc/client.config.ts, reached via
|
||||||
|
* the AuthClientService import chain.
|
||||||
|
*/
|
||||||
|
process.env.DUC_URL = process.env.DUC_URL || 'duc:localhost:0';
|
||||||
Reference in New Issue
Block a user