From 8e0182aa503dbe2f9eb2cfb29a2bcf74ec400292 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 24 Dec 2025 19:03:50 -0300 Subject: [PATCH 01/18] FEAT: add TOTP support for change password and local build support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass TOTP code to DUC for Keycloak users with MFA - Add Dockerfile.local for local protospack builds - Add .dockerignore to exclude node_modules from Docker context 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .dockerignore | 12 ++++++++++++ Dockerfile.local | 14 ++++++++++++-- src/modules/auth/auth.controller.ts | 3 ++- src/modules/auth/auth.service.ts | 2 ++ 4 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c9fc6a8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +node_modules +dist +.git +*.log +npm-debug.log* +.DS_Store +.env +.env.* +coverage +.nyc_output +*.tgz +!protospack.tgz diff --git a/Dockerfile.local b/Dockerfile.local index 03430e4..7f56186 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,4 +1,4 @@ -FROM node:20-alpine AS base_image +FROM node:22-alpine AS base_image RUN npm install -g npm@latest FROM base_image AS build_base @@ -19,10 +19,20 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ # Local build with secrets +# Check if local protospack tarball exists (for local dev) FROM build_base AS build +COPY protospack*.tgz* ./ RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \ aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 -RUN npm ci +RUN if [ -f "protospack.tgz" ]; then \ + echo "Installing from local protospack tarball..." && \ + npm install ./protospack.tgz --save-exact && \ + npm ci --ignore-scripts && \ + npm rebuild; \ + else \ + echo "Installing from CodeArtifact..." && \ + npm ci; \ + fi COPY . . RUN npm run build diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 3d86f14..3199fe3 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -190,13 +190,14 @@ export class AuthController { ) { this.logger.info('/auth - change-password'); - const { oldPassword, newPassword } = body; + const { oldPassword, newPassword, totpCode } = body; const { authorization: accessToken } = headers; return this.authClient.changePassword({ accessToken, oldPassword, newPassword, + totpCode, }); } diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index c1b85f6..6d67b95 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -143,6 +143,7 @@ export class AuthClientService implements OnModuleInit { accessToken, oldPassword, newPassword, + totpCode, }: AuthChangePasswordRequest) { this.logger.info('ChangePassword'); @@ -151,6 +152,7 @@ export class AuthClientService implements OnModuleInit { accessToken, oldPassword, newPassword, + totpCode, }), ); } From 55b0961b82bb18a04465d7d81774f4572fede40d Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Fri, 2 Jan 2026 09:21:46 -0300 Subject: [PATCH 02/18] FEAT: add organization info endpoints and DTOs for update and retrieval new organization forms --- package.json | 2 +- src/modules/customers/customers.controller.ts | 28 ++++++++++ src/modules/customers/customers.service.ts | 55 ++++++++++++++++++- src/modules/customers/dtos/organization.ts | 27 +++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 src/modules/customers/dtos/organization.ts diff --git a/package.json b/package.json index 2a2c4b8..575634b 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.19", + "@dadosfera/protospack-v2": "^3.38.0-beta.23", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index 2606ebd..b3f6c37 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -136,4 +136,32 @@ export class CustomersController { const result = await this.customersService.getAccessDashboardUrl(user.customer_name, metadata); return result; } + + @Get(':id/organization-info') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @ApiOkResponse({ description: 'Organization information' }) + async getOrganizationInfo(@Param('id') id: string) { + this.logger.info('getOrganizationInfo', { id }); + return this.customersService.getOrganizationInfo(id); + } + + @Put(':id/organization-info') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @HttpCode(HttpStatus.OK) + @ApiOkResponse({ description: 'Organization information updated' }) + async updateOrganizationInfo( + @Param('id') id: string, + @Body() body: { + name: string; + companySite: string; + domain: string; + cnpj: string; + description: string; + }, + ) { + return this.customersService.updateOrganizationInfo(id, body); + } + } diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index 8b28171..92b18ea 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -223,4 +223,57 @@ export class CustomersService implements OnModuleInit { }) ) } -} + + async updateOrganizationInfo( + customerId: string, + data: { + name: string; + companySite: string; + domain: string; + cnpj: string; + description: string; + }, + ) { + try { + const result = await lastValueFrom( + this.customerService.OrganizationUpdate({ + customerId, + name: data.name || '', + companySite: data.companySite || '', + domain: data.domain || '', + cnpj: data.cnpj || '', + description: data.description || '', + }), + ); + + return result; + } catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + else throw err; + } + } + + async getOrganizationInfo(customerId: string) { + try { + const customerResponse = await lastValueFrom( + this.customerService.CustomerFindOneById({ id: customerId }) + ); + + const customer = customerResponse.customer; + + return { + name: customer.name || '', + companySite: customer.companySite || '', + domain: customer.domain || '', + cnpj: customer.cnpj || '', + description: customer.description || '' + }; + } catch (err) { + if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) + throw new HttpException(err.details, HttpStatus.NOT_FOUND); + else throw err; + } + } + +} \ No newline at end of file diff --git a/src/modules/customers/dtos/organization.ts b/src/modules/customers/dtos/organization.ts new file mode 100644 index 0000000..50c3df7 --- /dev/null +++ b/src/modules/customers/dtos/organization.ts @@ -0,0 +1,27 @@ +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; + +export class OrganizationUpdateRequest { + @ApiProperty() + name: string; + @ApiPropertyOptional() + companySite: string; + @ApiProperty() + domain: string; + @ApiPropertyOptional() + info: string; + @ApiPropertyOptional() + cnpj: string; +} + +export class OrganizationResponse { + @ApiProperty() + name: string; + @ApiPropertyOptional() + companySite: string; + @ApiProperty() + domain: string; + @ApiPropertyOptional() + info: string; + @ApiPropertyOptional() + cnpj: string; +} \ No newline at end of file From da23ad76db6514b7044155e2fc7be2660aca2ab5 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Fri, 2 Jan 2026 09:22:27 -0300 Subject: [PATCH 03/18] DOCS: update docs to organization forms --- docsfera.json | 60 +++++++++++++++++++++++++++++++++++++++++++++++ package-lock.json | 9 ++++--- 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/docsfera.json b/docsfera.json index 29c24b7..157ccbb 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5461,6 +5461,66 @@ ] } }, + "/customers/{id}/organization-info": { + "get": { + "operationId": "CustomersController_getOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "put": { + "operationId": "CustomersController_updateOrganizationInfo", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Organization information updated" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/catalog/data-asset/share/{id}": { "get": { "operationId": "ShareController_getShareDataAsset", diff --git a/package-lock.json b/package-lock.json index 612a6b1..97c8c79 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.38.0-beta.19", + "@dadosfera/protospack-v2": "^3.38.0-beta.23", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,10 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.19", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.19.tgz", - "integrity": "sha512-qevGunl1IKv4POa2hhXNaDpqlh/bvnq5Hxo6534OHYPZ//S+XcRZgTrxFnylK3+uoROM8epF3Mey5GwynxY7Tw==", - "license": "ISC", + "version": "3.38.0-beta.23", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.23.tgz", + "integrity": "sha512-Juse1l1lleROO0uUrR5cGruGjkS/6NQau9v6eakvryv7FcLWJwgKV5qXMRkaTAMVTumAbxSqoKX8kGH5yZPKbw==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" From e2a7d2b92b51ac5e7d9228214957fabcf291a300 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 5 Jan 2026 11:47:20 -0300 Subject: [PATCH 04/18] FIX: update organization info field from 'name' to 'companyName' --- package-lock.json | 8 ++++---- package.json | 2 +- src/modules/customers/customers.service.ts | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 97c8c79..b2ab985 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.23", + "@dadosfera/protospack-v2": "^3.38.0-beta.24", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.23", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.23.tgz", - "integrity": "sha512-Juse1l1lleROO0uUrR5cGruGjkS/6NQau9v6eakvryv7FcLWJwgKV5qXMRkaTAMVTumAbxSqoKX8kGH5yZPKbw==", + "version": "3.38.0-beta.24", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.24.tgz", + "integrity": "sha512-RUDPY1WAy/eu+tN1sGMdYCBKt+UQ61dAjaiG+VyL/5WWP/vxcCW3QpFimSbFN04B8ZZW19a2V7UM61om8f4ywQ==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 575634b..010b6f8 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.23", + "@dadosfera/protospack-v2": "^3.38.0-beta.24", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index 92b18ea..602943b 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -227,7 +227,7 @@ export class CustomersService implements OnModuleInit { async updateOrganizationInfo( customerId: string, data: { - name: string; + companyName: string; companySite: string; domain: string; cnpj: string; @@ -238,7 +238,7 @@ export class CustomersService implements OnModuleInit { const result = await lastValueFrom( this.customerService.OrganizationUpdate({ customerId, - name: data.name || '', + companyName: data.companyName || '', companySite: data.companySite || '', domain: data.domain || '', cnpj: data.cnpj || '', @@ -263,7 +263,7 @@ export class CustomersService implements OnModuleInit { const customer = customerResponse.customer; return { - name: customer.name || '', + companyName: customer.companyName || '', companySite: customer.companySite || '', domain: customer.domain || '', cnpj: customer.cnpj || '', From 285de97375713b5e81ec0b26e7263f5d5cb69533 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 5 Jan 2026 12:01:04 -0300 Subject: [PATCH 05/18] FIX: update organization info field from 'name' to 'companyName' in updateOrganizationInfo method --- src/modules/customers/customers.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index b3f6c37..8ac5b3b 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -154,7 +154,7 @@ export class CustomersController { async updateOrganizationInfo( @Param('id') id: string, @Body() body: { - name: string; + companyName: string; companySite: string; domain: string; cnpj: string; From 8ac0a8a79f7d7209dbcda4f341a454415755a66f Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Mon, 5 Jan 2026 15:43:35 -0300 Subject: [PATCH 06/18] REFACTOR: rename 'companySite' to 'personalSite' in user DTOs and related services --- docsfera.json | 14 +++++++------- package-lock.json | 8 ++++---- package.json | 2 +- src/modules/users/dtos/entities.ts | 4 ++-- src/modules/users/users.service.ts | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docsfera.json b/docsfera.json index 157ccbb..572609f 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8229,7 +8229,7 @@ "companyName": { "type": "string" }, - "companySite": { + "personalSite": { "type": "string" }, "mfaStatus": { @@ -8264,7 +8264,7 @@ "hierarchy", "bio", "companyName", - "companySite", + "personalSite", "mfaStatus", "createdAt", "updatedAt", @@ -8404,7 +8404,7 @@ "companyName": { "type": "string" }, - "companySite": { + "personalSite": { "type": "string" }, "customer": { @@ -8436,7 +8436,7 @@ "hierarchy", "bio", "companyName", - "companySite", + "personalSite", "mfaStatus", "createdAt", "updatedAt", @@ -8580,7 +8580,7 @@ "bio": { "type": "string" }, - "companySite": { + "personalSite": { "type": "string" }, "companyName": { @@ -8632,7 +8632,7 @@ "companyName": { "type": "string" }, - "companySite": { + "personalSite": { "type": "string" }, "mfaStatus": { @@ -8660,7 +8660,7 @@ "hierarchy", "bio", "companyName", - "companySite", + "personalSite", "mfaStatus", "createdAt", "updatedAt", diff --git a/package-lock.json b/package-lock.json index b2ab985..e0de21d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.24", + "@dadosfera/protospack-v2": "^3.38.0-beta.25", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.24", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.24.tgz", - "integrity": "sha512-RUDPY1WAy/eu+tN1sGMdYCBKt+UQ61dAjaiG+VyL/5WWP/vxcCW3QpFimSbFN04B8ZZW19a2V7UM61om8f4ywQ==", + "version": "3.38.0-beta.25", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.25.tgz", + "integrity": "sha512-OsOl9hJezmtWlVy+2vl3qbWP5yHmLktJ01r5aL475lcfKJtzFPZieUT9ES87C1Emrqe21E6IM/BkJK2rm/aUiA==", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index 010b6f8..fa47b52 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.24", + "@dadosfera/protospack-v2": "^3.38.0-beta.25", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/users/dtos/entities.ts b/src/modules/users/dtos/entities.ts index 0f0e5a3..ab5e43c 100644 --- a/src/modules/users/dtos/entities.ts +++ b/src/modules/users/dtos/entities.ts @@ -43,7 +43,7 @@ export class User { @ApiProperty() companyName?: string; @ApiProperty() - companySite?: string; + personalSite?: string; @ApiPropertyOptional() customer?: Customer; @ApiProperty() @@ -117,7 +117,7 @@ export class UpdateUserReq { @ApiPropertyOptional() bio?: string; @ApiPropertyOptional() - companySite?: string; + personalSite?: string; @ApiPropertyOptional() companyName?: string; @ApiPropertyOptional() diff --git a/src/modules/users/users.service.ts b/src/modules/users/users.service.ts index 278188c..d402ff1 100644 --- a/src/modules/users/users.service.ts +++ b/src/modules/users/users.service.ts @@ -162,7 +162,7 @@ export class UsersService implements OnModuleInit { name: updateUserDTO.name, bio: updateUserDTO.bio, companyName: updateUserDTO.companyName, - companySite: updateUserDTO.companySite, + personalSite: updateUserDTO.personalSite, customerId, id, metabaseUserId: undefined, From 52bda8ebe24519ad6d1ebf85793c3dc967e076a4 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 7 Jan 2026 18:55:36 -0300 Subject: [PATCH 07/18] FEAT: add authProvider field to user response - Add authProvider to user entity DTO - Update user service to include authProvider - Update auth controller response Co-Authored-By: Claude Opus 4.5 --- docsfera.json | 12 ++++-------- package-lock.json | 9 +++++---- package.json | 2 +- src/modules/auth/auth.controller.ts | 3 ++- src/modules/users/dtos/entities.ts | 2 ++ src/modules/users/users.service.ts | 1 + 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/docsfera.json b/docsfera.json index 572609f..8942a26 100644 --- a/docsfera.json +++ b/docsfera.json @@ -213,14 +213,7 @@ ], "responses": { "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "object" - } - } - } + "description": "" } }, "tags": [ @@ -8252,6 +8245,9 @@ "items": { "type": "string" } + }, + "authProvider": { + "type": "string" } }, "required": [ diff --git a/package-lock.json b/package-lock.json index e0de21d..c27510d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.25", + "@dadosfera/protospack-v2": "^3.38.0-beta.26", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -1744,9 +1744,10 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.38.0-beta.25", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.25.tgz", - "integrity": "sha512-OsOl9hJezmtWlVy+2vl3qbWP5yHmLktJ01r5aL475lcfKJtzFPZieUT9ES87C1Emrqe21E6IM/BkJK2rm/aUiA==", + "version": "3.38.0-beta.26", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz", + "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==", + "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/package.json b/package.json index fa47b52..4fd10bd 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "@aws-sdk/signature-v4": "^3.370.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "^3.38.0-beta.25", + "@dadosfera/protospack-v2": "^3.38.0-beta.26", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 3199fe3..34dfd06 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -215,7 +215,8 @@ export class AuthController { const { username } = body; - return this.authClient.resetPassword({ username }, metadata); + await this.authClient.resetPassword({ username }, metadata); + return { authProvider: process.env.AUTH_PROVIDER || 'cognito' }; } @ApiInternalOnlyEndpoint() diff --git a/src/modules/users/dtos/entities.ts b/src/modules/users/dtos/entities.ts index ab5e43c..eef5ca0 100644 --- a/src/modules/users/dtos/entities.ts +++ b/src/modules/users/dtos/entities.ts @@ -64,6 +64,8 @@ export class UserNoRolesAndCustomer extends OmitType(UserNoRoles, [ export class IUserByCustomer extends OmitType(User, ['customer']) { @ApiPropertyOptional() permissions?: string[]; + @ApiPropertyOptional() + authProvider?: string; } export class CreateUserReq { diff --git a/src/modules/users/users.service.ts b/src/modules/users/users.service.ts index d402ff1..29ec0e3 100644 --- a/src/modules/users/users.service.ts +++ b/src/modules/users/users.service.ts @@ -125,6 +125,7 @@ export class UsersService implements OnModuleInit { return { permissions }; }); res.user.permissions = permissions; + res.user.authProvider = process.env.AUTH_PROVIDER || 'cognito'; return res; } From 3b8310fdca45be66a74d525752056be9641bc147 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 7 Jan 2026 19:04:07 -0300 Subject: [PATCH 08/18] CHORE: simplify Dockerfile.local to use npm ci only Remove protospack tarball fallback logic Co-Authored-By: Claude Opus 4.5 --- Dockerfile.local | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/Dockerfile.local b/Dockerfile.local index 7f56186..fc8afaf 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -19,20 +19,10 @@ ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ # Local build with secrets -# Check if local protospack tarball exists (for local dev) FROM build_base AS build -COPY protospack*.tgz* ./ RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \ aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 -RUN if [ -f "protospack.tgz" ]; then \ - echo "Installing from local protospack tarball..." && \ - npm install ./protospack.tgz --save-exact && \ - npm ci --ignore-scripts && \ - npm rebuild; \ - else \ - echo "Installing from CodeArtifact..." && \ - npm ci; \ - fi +RUN npm ci COPY . . RUN npm run build From 1881d07c4a5d9ba71c45eb00d808f779cfcce1ed Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 8 Jan 2026 12:10:17 -0300 Subject: [PATCH 09/18] FEAT: add dual auth provider support to Helm chart - Add auth_provider config to values.yaml (cognito/keycloak) - Add AUTH_PROVIDER env var to deployment template - Add conditional Keycloak env vars (URL, realm, client ID, secret) - Enables parallel deployments with different auth providers Co-Authored-By: Claude Opus 4.5 --- deploy/helm-chart/templates/deployment.yaml | 17 +++++++++++++++++ deploy/helm-chart/values.yaml | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index 7312b30..fa44bc1 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -48,10 +48,27 @@ spec: {{- toYaml .Values.resources | nindent 12 }} {{- end }} env: + # Auth Provider Configuration + - name: AUTH_PROVIDER + value: {{ .Values.maestro.auth_provider | default "cognito" | quote }} - name: AWS_IDENTITY_POOL_ID value: {{ .Values.maestro.aws_identity_pool_id }} - name: AWS_REGION value: "us-east-1" + {{- if eq .Values.maestro.auth_provider "keycloak" }} + # Keycloak Configuration (when auth_provider=keycloak) + - name: KEYCLOAK_URL + value: {{ .Values.maestro.keycloak_url | quote }} + - name: KEYCLOAK_REALM + value: {{ .Values.maestro.keycloak_realm | default "dadosfera" | quote }} + - name: KEYCLOAK_CLIENT_ID + value: {{ .Values.maestro.keycloak_client_id | quote }} + - name: KEYCLOAK_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.maestro.keycloak_secret_name | default "maestro-keycloak-credentials" }} + key: client-secret + {{- end }} - name: BASE_HOST value: "maestro_prd" - name: BUCKET_CUSTOMER_CSV_ASSETS diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 30af31e..8730481 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -27,7 +27,14 @@ resources: cpu: 2000m memory: 2Gi maestro: + # Auth provider: "cognito" (default) or "keycloak" + auth_provider: "cognito" aws_identity_pool_id: "us-east-1_Mrezsw9Sn" + # Keycloak config (when auth_provider=keycloak) + keycloak_url: "" + keycloak_realm: "dadosfera" + keycloak_client_id: "" + keycloak_secret_name: "maestro-keycloak-credentials" duc_url: duc.dadosfera.ai in_factory_url: in-factory.dadosfera.ai bucket_customer_csv_assets: "customers-csv-assets-prd-611330257153" From 8c34914806fc2e7d61cf94ea464e9d5c1342265d Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 8 Jan 2026 12:13:59 -0300 Subject: [PATCH 10/18] FEAT: add AUTH_PROVIDER config to Helm chart - Add auth_provider to values.yaml (cognito/keycloak) - Add AUTH_PROVIDER env var to deployment template - Note: maestro only needs to know which provider is used, duc handles connection Co-Authored-By: Claude Opus 4.5 --- deploy/helm-chart/templates/deployment.yaml | 16 +--------------- deploy/helm-chart/values.yaml | 6 +----- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index fa44bc1..dc8949d 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -48,27 +48,13 @@ spec: {{- toYaml .Values.resources | nindent 12 }} {{- end }} env: - # Auth Provider Configuration + # Auth Provider Configuration (cognito or keycloak) - name: AUTH_PROVIDER value: {{ .Values.maestro.auth_provider | default "cognito" | quote }} - name: AWS_IDENTITY_POOL_ID value: {{ .Values.maestro.aws_identity_pool_id }} - name: AWS_REGION value: "us-east-1" - {{- if eq .Values.maestro.auth_provider "keycloak" }} - # Keycloak Configuration (when auth_provider=keycloak) - - name: KEYCLOAK_URL - value: {{ .Values.maestro.keycloak_url | quote }} - - name: KEYCLOAK_REALM - value: {{ .Values.maestro.keycloak_realm | default "dadosfera" | quote }} - - name: KEYCLOAK_CLIENT_ID - value: {{ .Values.maestro.keycloak_client_id | quote }} - - name: KEYCLOAK_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .Values.maestro.keycloak_secret_name | default "maestro-keycloak-credentials" }} - key: client-secret - {{- end }} - name: BASE_HOST value: "maestro_prd" - name: BUCKET_CUSTOMER_CSV_ASSETS diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 8730481..37cd6be 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -28,13 +28,9 @@ resources: memory: 2Gi maestro: # Auth provider: "cognito" (default) or "keycloak" + # Note: maestro doesn't connect to Keycloak directly, only duc does auth_provider: "cognito" aws_identity_pool_id: "us-east-1_Mrezsw9Sn" - # Keycloak config (when auth_provider=keycloak) - keycloak_url: "" - keycloak_realm: "dadosfera" - keycloak_client_id: "" - keycloak_secret_name: "maestro-keycloak-credentials" duc_url: duc.dadosfera.ai in_factory_url: in-factory.dadosfera.ai bucket_customer_csv_assets: "customers-csv-assets-prd-611330257153" From 85c8a4937d01eb38c0a98b2f8ae95f8494bb0a1f Mon Sep 17 00:00:00 2001 From: Rafael Date: Mon, 12 Jan 2026 16:22:25 -0300 Subject: [PATCH 11/18] FIX: Update ValidationPipe for class-validator 0.14.0+ compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fix addresses a breaking change introduced in class-validator 0.14.0 where the default for forbidUnknownValues changed from false to true. Issue: - POST /connections was returning 400 "an unknown value was passed to the validate function" errors - This occurred because CreateConnectionDto and UpdateConnectionDto have no validation decorators, causing class-validator 0.14.0+ to treat them as "unknown values" - Extra fields (like connector_version) in request payloads would fail validation Solution: - Set forbidUnknownValues: false to allow DTOs without validation decorators - Set whitelist: true to automatically strip extra properties not defined in DTOs - This maintains backward compatibility while adding security by removing unexpected fields 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/pipes/object-validation.pipe.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pipes/object-validation.pipe.ts b/src/pipes/object-validation.pipe.ts index 1014581..1315083 100644 --- a/src/pipes/object-validation.pipe.ts +++ b/src/pipes/object-validation.pipe.ts @@ -14,7 +14,10 @@ export class ValidationPipe implements PipeTransform { return value; } const object = plainToInstance(metatype, value); - const errors = await validate(object); + const errors = await validate(object, { + forbidUnknownValues: false, + whitelist: true, + }); if (errors.length > 0) { const errorMessages = errors.map((err) => err.constraints); throw new BadRequestException(errorMessages); From a2c7ce00db92d11eb674d62d82006f64a937a280 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 13 Jan 2026 10:29:38 -0300 Subject: [PATCH 12/18] FEAT: integrate Storage Explorer API with new endpoints and permissions --- docsfera.json | 751 ++++++++++++++++++ package-lock.json | 1 - src/app.module.ts | 2 + src/authentication/permissions.enum.ts | 29 + .../storage-explorer.config.ts | 13 + .../storage-explorer.controller.ts | 384 +++++++++ .../storage-explorer.module.ts | 13 + .../storage-explorer.service.ts | 190 +++++ 8 files changed, 1382 insertions(+), 1 deletion(-) create mode 100644 src/modules/storage-explorer/storage-explorer.config.ts create mode 100644 src/modules/storage-explorer/storage-explorer.controller.ts create mode 100644 src/modules/storage-explorer/storage-explorer.module.ts create mode 100644 src/modules/storage-explorer/storage-explorer.service.ts diff --git a/docsfera.json b/docsfera.json index 8942a26..92b3f98 100644 --- a/docsfera.json +++ b/docsfera.json @@ -7767,6 +7767,757 @@ ] } }, + "/storage-explorer/tables/validate-name": { + "post": { + "operationId": "StorageExplorerController_validateTableName", + "summary": "Validate table name in PostgreSQL and Snowflake", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables": { + "post": { + "operationId": "StorageExplorerController_createTable", + "summary": "Create a new table", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "get": { + "operationId": "StorageExplorerController_listTables", + "summary": "List all tables with pagination", + "parameters": [ + { + "name": "page", + "required": true, + "in": "query", + "schema": { + "type": "number" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}": { + "get": { + "operationId": "StorageExplorerController_getTable", + "summary": "Get table details by ID", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}/datasets/{datasetId}": { + "post": { + "operationId": "StorageExplorerController_linkDatasetToTable", + "summary": "Link a dataset to a table", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}/datasets": { + "get": { + "operationId": "StorageExplorerController_getTableDatasets", + "summary": "Get all datasets linked to a table", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}/schema": { + "get": { + "operationId": "StorageExplorerController_getTableSchema", + "summary": "Get table schema", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/tables/{tableId}/validate-compatibility/{datasetId}": { + "post": { + "operationId": "StorageExplorerController_validateSchemaCompatibility", + "summary": "Validate schema compatibility between table and dataset", + "parameters": [ + { + "name": "tableId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/datasets/{datasetId}/preview": { + "get": { + "operationId": "StorageExplorerController_getDatasetPreview", + "summary": "Get dataset preview data", + "parameters": [ + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "required": true, + "in": "query", + "schema": { + "type": "number" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/datasets/{datasetId}/schema": { + "get": { + "operationId": "StorageExplorerController_getDatasetSchema", + "summary": "Get dataset schema information", + "parameters": [ + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + }, + { + "name": "force_refresh", + "required": true, + "in": "query", + "schema": { + "type": "boolean" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/datasets/upload/{uploadId}": { + "get": { + "operationId": "StorageExplorerController_listDatasetsByUpload", + "summary": "List all datasets for a specific upload", + "parameters": [ + { + "name": "uploadId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/datasets/{datasetId}/refresh-schema": { + "put": { + "operationId": "StorageExplorerController_refreshDatasetSchema", + "summary": "Refresh dataset schema with new parsing options (Excel)", + "parameters": [ + { + "name": "datasetId", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/uploads/history": { + "get": { + "operationId": "StorageExplorerController_listFileExplorerUploads", + "summary": "List file explorer uploads with pagination", + "parameters": [ + { + "name": "page", + "required": true, + "in": "query", + "schema": { + "type": "number" + } + }, + { + "name": "limit", + "required": true, + "in": "query", + "schema": { + "type": "number" + } + }, + { + "name": "folder_path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/browse": { + "get": { + "operationId": "StorageExplorerController_browseStorage", + "summary": "Browse folders and files in storage", + "parameters": [ + { + "name": "path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/upload/batch": { + "post": { + "operationId": "StorageExplorerController_batchUpload", + "summary": "Upload multiple files to storage", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/folder/create": { + "post": { + "operationId": "StorageExplorerController_createFolder", + "summary": "Create a new folder in storage", + "parameters": [], + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/download": { + "get": { + "operationId": "StorageExplorerController_downloadFile", + "summary": "Download a file from storage", + "parameters": [ + { + "name": "file_path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/delete": { + "delete": { + "operationId": "StorageExplorerController_deleteFile", + "summary": "Delete a file from storage", + "parameters": [ + { + "name": "file_path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/storage-explorer/storage/metadata": { + "get": { + "operationId": "StorageExplorerController_getFileMetadata", + "summary": "Get detailed file metadata", + "parameters": [ + { + "name": "file_path", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Storage Explorer" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/health": { "get": { "operationId": "HealthController_check", diff --git a/package-lock.json b/package-lock.json index c27510d..4d90eb7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1747,7 +1747,6 @@ "version": "3.38.0-beta.26", "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.26.tgz", "integrity": "sha512-N8NS7+djLGy0wJXk00+4oupqd/wBIQ1f+YBBhK2y9x4guFXYK1KWIrPZhPj+gaU8g2KNkqKoT7SnE9PXNxlLSQ==", - "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", "rxjs": "^7.5.5" diff --git a/src/app.module.ts b/src/app.module.ts index 27d042c..0865ad2 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -34,6 +34,7 @@ import { AssignModule } from './modules/assign/assign.module'; import { ShareMetadataModule } from './modules/share-metadata/share-metadata.module'; import { ApiKeyModule } from './modules/api-key/api-key.module'; import { PlatformApiModule } from './modules/platform-api/platform-api.module'; +import { StorageExplorerModule } from './modules/storage-explorer/storage-explorer.module'; @Module({ providers: [ @@ -75,6 +76,7 @@ import { PlatformApiModule } from './modules/platform-api/platform-api.module'; IdentityProviderModule, NetworkPolicyModule, PlatformApiModule, + StorageExplorerModule, //Always leave HealthModule last, so it is on the bottom of swagger HealthModule, ], diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index 3eeb2a7..ca5e9ac 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -668,6 +668,35 @@ export const PERMISSIONS_GROUPS = { }, }, }, + STORAGE_EXPLORER: { + title: { + 'pt-br': 'Storage Explorer', + 'en-us': 'Storage Explorer', + 'es-es': 'Storage Explorer', + }, + permissions: { + READ: { + seqid: 51, + claim: 'storage-explorer:read', + usage: PermissionUsages.PUBLIC, + name: { + 'pt-br': 'Ler dados do Storage Explorer', + 'en-us': 'Read Storage Explorer data', + 'es-es': 'Leer datos del Storage Explorer', + }, + }, + WRITE: { + seqid: 52, + claim: 'storage-explorer:write', + usage: PermissionUsages.PUBLIC, + name: { + 'pt-br': 'Escrever dados no Storage Explorer', + 'en-us': 'Write Storage Explorer data', + 'es-es': 'Escribir datos en Storage Explorer', + }, + }, + }, + }, }; export interface DadosferaModule { name: string; diff --git a/src/modules/storage-explorer/storage-explorer.config.ts b/src/modules/storage-explorer/storage-explorer.config.ts new file mode 100644 index 0000000..417eb15 --- /dev/null +++ b/src/modules/storage-explorer/storage-explorer.config.ts @@ -0,0 +1,13 @@ +export const STORAGE_EXPLORER_CONFIG = { + getUrl: (customerId: string): string => { + const urlTemplate = process.env.STORAGE_EXPLORER_API_URL; + if (!urlTemplate) { + throw new Error('STORAGE_EXPLORER_API_URL environment variable is not set'); + } + // Replace {customer_id} placeholder with actual customer ID + // For local: http://172.17.0.1:8000/api (no placeholder) + // For prod: https://storage-explorer-{customer_id}.stg.dadosfera.ai/api + return urlTemplate.replace('{customer_id}', customerId); + }, + timeout: parseInt(process.env.STORAGE_EXPLORER_TIMEOUT || '30000', 10), +}; diff --git a/src/modules/storage-explorer/storage-explorer.controller.ts b/src/modules/storage-explorer/storage-explorer.controller.ts new file mode 100644 index 0000000..a18f4a9 --- /dev/null +++ b/src/modules/storage-explorer/storage-explorer.controller.ts @@ -0,0 +1,384 @@ +import { + Controller, + Get, + Post, + Put, + Delete, + Param, + Body, + Query, + Inject, + UseInterceptors, + UploadedFiles, +} from '@nestjs/common'; +import { ApiTags, ApiOperation, ApiConsumes } from '@nestjs/swagger'; +import { FilesInterceptor } from '@nestjs/platform-express'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; +import FormData from 'form-data'; + +import { + Authenticated, + RequireAllPermissions, +} from '../../decorators/authentication.decorator'; +import { User, RequestUser } from '../../decorators/user.decorator'; +import { StorageExplorerService } from './storage-explorer.service'; +import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; + +@ApiTags('Storage Explorer') +@Controller('storage-explorer') +export class StorageExplorerController { + private logger: any; + + constructor( + private readonly storageExplorerService: StorageExplorerService, + @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + // ============================================ + // TABLE OPERATIONS + // ============================================ + + @ApiOperation({ summary: 'Validate table name in PostgreSQL and Snowflake' }) + @Post('tables/validate-name') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async validateTableName( + @Body() body: any, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + '/tables/validate-name', + user, + body, + ); + } + + @ApiOperation({ summary: 'Create a new table' }) + @Post('tables') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async createTable( + @Body() body: any, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + '/tables/', + user, + body, + ); + } + + @ApiOperation({ summary: 'List all tables with pagination' }) + @Get('tables') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async listTables( + @Query('page') page: number, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/tables/', + user, + undefined, + { page }, + ); + } + + @ApiOperation({ summary: 'Get table details by ID' }) + @Get('tables/:tableId') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getTable( + @Param('tableId') tableId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/tables/${tableId}`, + user, + ); + } + + @ApiOperation({ summary: 'Link a dataset to a table' }) + @Post('tables/:tableId/datasets/:datasetId') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async linkDatasetToTable( + @Param('tableId') tableId: string, + @Param('datasetId') datasetId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + `/tables/${tableId}/datasets/${datasetId}`, + user, + ); + } + + @ApiOperation({ summary: 'Get all datasets linked to a table' }) + @Get('tables/:tableId/datasets') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getTableDatasets( + @Param('tableId') tableId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/tables/${tableId}/datasets`, + user, + ); + } + + @ApiOperation({ summary: 'Get table schema' }) + @Get('tables/:tableId/schema') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getTableSchema( + @Param('tableId') tableId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/tables/${tableId}/schema`, + user, + ); + } + + @ApiOperation({ summary: 'Validate schema compatibility between table and dataset' }) + @Post('tables/:tableId/validate-compatibility/:datasetId') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async validateSchemaCompatibility( + @Param('tableId') tableId: string, + @Param('datasetId') datasetId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + `/tables/${tableId}/validate-compatibility/${datasetId}`, + user, + ); + } + + // ============================================ + // DATASET OPERATIONS + // ============================================ + + @ApiOperation({ summary: 'Get dataset preview data' }) + @Get('datasets/:datasetId/preview') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getDatasetPreview( + @Param('datasetId') datasetId: string, + @Query('limit') limit: number, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/datasets/${datasetId}/preview`, + user, + undefined, + { limit }, + ); + } + + @ApiOperation({ summary: 'Get dataset schema information' }) + @Get('datasets/:datasetId/schema') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getDatasetSchema( + @Param('datasetId') datasetId: string, + @Query('force_refresh') forceRefresh: boolean, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/datasets/${datasetId}/schema`, + user, + undefined, + { force_refresh: forceRefresh }, + ); + } + + @ApiOperation({ summary: 'List all datasets for a specific upload' }) + @Get('datasets/upload/:uploadId') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async listDatasetsByUpload( + @Param('uploadId') uploadId: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + `/datasets/upload/${uploadId}`, + user, + ); + } + + @ApiOperation({ summary: 'Refresh dataset schema with new parsing options (Excel)' }) + @Put('datasets/:datasetId/refresh-schema') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async refreshDatasetSchema( + @Param('datasetId') datasetId: string, + @Body() body: any, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'PUT', + `/datasets/${datasetId}/refresh-schema`, + user, + body, + ); + } + + // ============================================ + // STORAGE OPERATIONS + // ============================================ + + @ApiOperation({ summary: 'List file explorer uploads with pagination' }) + @Get('storage/uploads/history') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async listFileExplorerUploads( + @Query('page') page: number, + @Query('limit') limit: number, + @Query('folder_path') folderPath: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/storage/uploads/history', + user, + undefined, + { page, limit, folder_path: folderPath }, + ); + } + + @ApiOperation({ summary: 'Browse folders and files in storage' }) + @Get('storage/browse') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async browseStorage( + @Query('path') path: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/storage/browse', + user, + undefined, + { path }, + ); + } + + @ApiOperation({ summary: 'Upload multiple files to storage' }) + @Post('storage/upload/batch') + @ApiConsumes('multipart/form-data') + @UseInterceptors(FilesInterceptor('files')) + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async batchUpload( + @UploadedFiles() files: Array, + @Body('folder_path') folderPath: string, + @User() user: RequestUser, + ) { + // Create FormData to forward files to storage-explorer API + const formData = new FormData(); + + // Add files + if (files && files.length > 0) { + files.forEach((file) => { + formData.append('files', file.buffer, { + filename: file.originalname, + contentType: file.mimetype, + }); + }); + } + + // Add folder_path + if (folderPath) { + formData.append('folder_path', folderPath); + } + + return this.storageExplorerService.proxyFormData( + 'POST', + '/storage/upload/batch', + user, + formData, + ); + } + + @ApiOperation({ summary: 'Create a new folder in storage' }) + @Post('storage/folder/create') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async createFolder( + @Body() body: any, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'POST', + '/storage/folder/create', + user, + body, + ); + } + + @ApiOperation({ summary: 'Download a file from storage' }) + @Get('storage/download') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async downloadFile( + @Query('file_path') filePath: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/storage/download', + user, + undefined, + { file_path: filePath }, + ); + } + + @ApiOperation({ summary: 'Delete a file from storage' }) + @Delete('storage/delete') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) + async deleteFile( + @Query('file_path') filePath: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'DELETE', + '/storage/delete', + user, + undefined, + { file_path: filePath }, + ); + } + + @ApiOperation({ summary: 'Get detailed file metadata' }) + @Get('storage/metadata') + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) + async getFileMetadata( + @Query('file_path') filePath: string, + @User() user: RequestUser, + ) { + return this.storageExplorerService.proxy( + 'GET', + '/storage/metadata', + user, + undefined, + { file_path: filePath }, + ); + } +} diff --git a/src/modules/storage-explorer/storage-explorer.module.ts b/src/modules/storage-explorer/storage-explorer.module.ts new file mode 100644 index 0000000..b0f0e86 --- /dev/null +++ b/src/modules/storage-explorer/storage-explorer.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { StorageExplorerController } from './storage-explorer.controller'; +import { StorageExplorerService } from './storage-explorer.service'; + +@Module({ + imports: [], + controllers: [StorageExplorerController], + providers: [StorageExplorerService, DadosferaLogger], + exports: [StorageExplorerService], +}) +export class StorageExplorerModule {} diff --git a/src/modules/storage-explorer/storage-explorer.service.ts b/src/modules/storage-explorer/storage-explorer.service.ts new file mode 100644 index 0000000..1430dbb --- /dev/null +++ b/src/modules/storage-explorer/storage-explorer.service.ts @@ -0,0 +1,190 @@ +import { Injectable, Inject, HttpException } from '@nestjs/common'; +import axios, { AxiosResponse, Method } from 'axios'; +import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; + +import { RequestUser } from '../../decorators/user.decorator'; +import { STORAGE_EXPLORER_CONFIG } from './storage-explorer.config'; + +@Injectable() +export class StorageExplorerService { + private logger: any; + + constructor( + @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + async proxy( + method: string, + path: string, + user: RequestUser, + body?: any, + query?: Record, + ): Promise { + // Validate customer_id is present for multi-tenant isolation + if (!user.customer_id) { + throw new HttpException('Customer ID is required for storage operations', 400); + } + + // Get customer-specific storage-explorer URL + const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id); + const url = new URL(`${baseUrl}${path}`); + + // Add query params + if (query) { + Object.entries(query).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)); + } + }); + } + + const headers: Record = { + 'content-type': 'application/json', + // Forward user context for logging + 'x-customer-id': user.customer_id, + 'x-customer-name': user.customer_name || '', + 'x-user-id': user.user_id || '', + 'x-username': user.username || '', + 'x-customer-tier': user.customer_tier || '', + }; + + this.logger.info('Proxying request to storage-explorer', { + method: method.toUpperCase(), + path, + customer_id: user.customer_id, + storage_url: baseUrl, + user_id: user.user_id, + }); + + try { + const response: AxiosResponse = await axios({ + method: method as Method, + url: url.href, + headers, + data: body, + timeout: STORAGE_EXPLORER_CONFIG.timeout, + validateStatus: () => true, // Don't throw on non-2xx + }); + + // Propagate non-2xx responses as HttpExceptions + if (response.status >= 400) { + throw new HttpException(response.data, response.status); + } + + return response.data; + } catch (error) { + this.logger.error('Storage Explorer API proxy error', { + error: error.message, + status: error.response?.status, + path, + storage_url: baseUrl, + method: method.toUpperCase(), + }); + + if (error instanceof HttpException) { + throw error; + } + + if (error.response) { + throw new HttpException(error.response.data, error.response.status); + } + + if (error.code === 'ECONNREFUSED') { + throw new HttpException('Storage Explorer API service unavailable', 503); + } + + if (error.code === 'ETIMEDOUT' || error.code === 'ECONNABORTED') { + throw new HttpException('Storage Explorer API request timeout', 504); + } + + throw new HttpException('Internal server error', 500); + } + } + + /** + * Proxy with file upload support (multipart/form-data) + */ + async proxyFormData( + method: string, + path: string, + user: RequestUser, + formData: any, + query?: Record, + ): Promise { + // Validate customer_id is present for multi-tenant isolation + if (!user.customer_id) { + throw new HttpException('Customer ID is required for storage operations', 400); + } + + // Get customer-specific storage-explorer URL + const baseUrl = STORAGE_EXPLORER_CONFIG.getUrl(user.customer_id); + const url = new URL(`${baseUrl}${path}`); + + // Add query params + if (query) { + Object.entries(query).forEach(([key, value]) => { + if (value !== undefined && value !== null) { + url.searchParams.set(key, String(value)); + } + }); + } + + const headers: Record = { + // Forward user context for logging + 'x-customer-id': user.customer_id, + 'x-customer-name': user.customer_name || '', + 'x-user-id': user.user_id || '', + 'x-username': user.username || '', + 'x-customer-tier': user.customer_tier || '', + // Let axios set Content-Type for multipart/form-data with boundary + ...formData.getHeaders?.(), + }; + + this.logger.info('Proxying form data request to storage-explorer', { + method: method.toUpperCase(), + path, + customer_id: user.customer_id, + storage_url: baseUrl, + user_id: user.user_id, + }); + + try { + const response: AxiosResponse = await axios({ + method: method as Method, + url: url.href, + headers, + data: formData, + timeout: STORAGE_EXPLORER_CONFIG.timeout, + maxContentLength: Infinity, + maxBodyLength: Infinity, + validateStatus: () => true, + }); + + if (response.status >= 400) { + throw new HttpException(response.data, response.status); + } + + return response.data; + } catch (error) { + this.logger.error('Storage Explorer API form data proxy error', { + error: error.message, + status: error.response?.status, + path, + storage_url: baseUrl, + method: method.toUpperCase(), + }); + + if (error instanceof HttpException) { + throw error; + } + + if (error.response) { + throw new HttpException(error.response.data, error.response.status); + } + + throw new HttpException('Internal server error', 500); + } + } +} From 0ce382230009338f72c04e4108004579cf6487c0 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Tue, 13 Jan 2026 14:36:43 -0300 Subject: [PATCH 13/18] CHORE: update new endpoint link-table and remove /delete from storage-explorer api --- .../storage-explorer.controller.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/modules/storage-explorer/storage-explorer.controller.ts b/src/modules/storage-explorer/storage-explorer.controller.ts index a18f4a9..34884da 100644 --- a/src/modules/storage-explorer/storage-explorer.controller.ts +++ b/src/modules/storage-explorer/storage-explorer.controller.ts @@ -348,20 +348,19 @@ export class StorageExplorerController { ); } - @ApiOperation({ summary: 'Delete a file from storage' }) - @Delete('storage/delete') + @ApiOperation({ summary: 'Link an existing file in storage to a table' }) + @Post('storage/link-to-table') @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) - async deleteFile( - @Query('file_path') filePath: string, + async linkFileToTable( + @Body() body: any, @User() user: RequestUser, ) { return this.storageExplorerService.proxy( - 'DELETE', - '/storage/delete', + 'POST', + '/storage/link-to-table', user, - undefined, - { file_path: filePath }, + body, ); } From 3b844090038c9d6b2aa6d551cc8a3c4709999e41 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 14 Jan 2026 06:25:58 -0300 Subject: [PATCH 14/18] CHORE: add unique authentication decorator to Storage Explorer controller methods --- .../storage-explorer.controller.ts | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/modules/storage-explorer/storage-explorer.controller.ts b/src/modules/storage-explorer/storage-explorer.controller.ts index 34884da..fd2e4d9 100644 --- a/src/modules/storage-explorer/storage-explorer.controller.ts +++ b/src/modules/storage-explorer/storage-explorer.controller.ts @@ -26,6 +26,7 @@ import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; @ApiTags('Storage Explorer') @Controller('storage-explorer') +@Authenticated() export class StorageExplorerController { private logger: any; @@ -42,7 +43,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Validate table name in PostgreSQL and Snowflake' }) @Post('tables/validate-name') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async validateTableName( @Body() body: any, @@ -58,7 +58,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Create a new table' }) @Post('tables') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async createTable( @Body() body: any, @@ -74,7 +73,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'List all tables with pagination' }) @Get('tables') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async listTables( @Query('page') page: number, @@ -91,7 +89,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get table details by ID' }) @Get('tables/:tableId') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getTable( @Param('tableId') tableId: string, @@ -106,7 +103,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Link a dataset to a table' }) @Post('tables/:tableId/datasets/:datasetId') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async linkDatasetToTable( @Param('tableId') tableId: string, @@ -122,7 +118,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get all datasets linked to a table' }) @Get('tables/:tableId/datasets') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getTableDatasets( @Param('tableId') tableId: string, @@ -137,7 +132,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get table schema' }) @Get('tables/:tableId/schema') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getTableSchema( @Param('tableId') tableId: string, @@ -152,7 +146,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Validate schema compatibility between table and dataset' }) @Post('tables/:tableId/validate-compatibility/:datasetId') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async validateSchemaCompatibility( @Param('tableId') tableId: string, @@ -172,7 +165,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get dataset preview data' }) @Get('datasets/:datasetId/preview') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getDatasetPreview( @Param('datasetId') datasetId: string, @@ -190,7 +182,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get dataset schema information' }) @Get('datasets/:datasetId/schema') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getDatasetSchema( @Param('datasetId') datasetId: string, @@ -208,7 +199,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'List all datasets for a specific upload' }) @Get('datasets/upload/:uploadId') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async listDatasetsByUpload( @Param('uploadId') uploadId: string, @@ -223,7 +213,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Refresh dataset schema with new parsing options (Excel)' }) @Put('datasets/:datasetId/refresh-schema') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async refreshDatasetSchema( @Param('datasetId') datasetId: string, @@ -244,7 +233,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'List file explorer uploads with pagination' }) @Get('storage/uploads/history') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async listFileExplorerUploads( @Query('page') page: number, @@ -263,7 +251,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Browse folders and files in storage' }) @Get('storage/browse') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async browseStorage( @Query('path') path: string, @@ -282,7 +269,6 @@ export class StorageExplorerController { @Post('storage/upload/batch') @ApiConsumes('multipart/form-data') @UseInterceptors(FilesInterceptor('files')) - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async batchUpload( @UploadedFiles() files: Array, @@ -317,7 +303,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Create a new folder in storage' }) @Post('storage/folder/create') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async createFolder( @Body() body: any, @@ -333,7 +318,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Download a file from storage' }) @Get('storage/download') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async downloadFile( @Query('file_path') filePath: string, @@ -350,7 +334,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Link an existing file in storage to a table' }) @Post('storage/link-to-table') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) async linkFileToTable( @Body() body: any, @@ -366,7 +349,6 @@ export class StorageExplorerController { @ApiOperation({ summary: 'Get detailed file metadata' }) @Get('storage/metadata') - @Authenticated() @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) async getFileMetadata( @Query('file_path') filePath: string, From 8e63757738524fd85e9303be1cbfc4256f7518f3 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 14 Jan 2026 06:26:21 -0300 Subject: [PATCH 15/18] FEAT: add STORAGE_EXPLORER_API_URL to deployment and values configuration --- deploy/helm-chart/templates/deployment.yaml | 2 ++ deploy/helm-chart/values.yaml | 1 + 2 files changed, 3 insertions(+) diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml index dc8949d..68b8e1d 100644 --- a/deploy/helm-chart/templates/deployment.yaml +++ b/deploy/helm-chart/templates/deployment.yaml @@ -111,6 +111,8 @@ spec: value: "{{ .Values.maestro.redis_tls }}" - name: PLATFORM_API_URL value: {{ .Values.maestro.platform_api_url }} + - name: STORAGE_EXPLORER_API_URL + value: {{ .Values.maestro.storage_explorer_api_url | quote }} - name: JWT_PRIVATE_KEY valueFrom: secretKeyRef: diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml index 37cd6be..3c8ebcc 100644 --- a/deploy/helm-chart/values.yaml +++ b/deploy/helm-chart/values.yaml @@ -47,6 +47,7 @@ maestro: open_customer_id: f239718a-a271-4ef9-ae7e-02a2f0f3aa6e open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9 platform_api_url: https://oz8v2zid1e.execute-api.us-east-1.amazonaws.com + storage_explorer_api_url: "https://storage-explorer-{customer_id}.prd.dadosfera.ai/api" dedicated_proxy: "" restricted_ip: "" redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com" From b44d23552c2ee1b4e681f3f4b8f5f6ecb735809c Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 14 Jan 2026 06:40:52 -0300 Subject: [PATCH 16/18] CHORE: remove unecessary header for request --- docsfera.json | 21 ++++++------------- .../storage-explorer.service.ts | 12 ----------- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/docsfera.json b/docsfera.json index 92b3f98..ec330d1 100644 --- a/docsfera.json +++ b/docsfera.json @@ -8440,22 +8440,13 @@ ] } }, - "/storage-explorer/storage/delete": { - "delete": { - "operationId": "StorageExplorerController_deleteFile", - "summary": "Delete a file from storage", - "parameters": [ - { - "name": "file_path", - "required": true, - "in": "query", - "schema": { - "type": "string" - } - } - ], + "/storage-explorer/storage/link-to-table": { + "post": { + "operationId": "StorageExplorerController_linkFileToTable", + "summary": "Link an existing file in storage to a table", + "parameters": [], "responses": { - "200": { + "201": { "description": "", "content": { "application/json": { diff --git a/src/modules/storage-explorer/storage-explorer.service.ts b/src/modules/storage-explorer/storage-explorer.service.ts index 1430dbb..ba149c0 100644 --- a/src/modules/storage-explorer/storage-explorer.service.ts +++ b/src/modules/storage-explorer/storage-explorer.service.ts @@ -42,12 +42,6 @@ export class StorageExplorerService { const headers: Record = { 'content-type': 'application/json', - // Forward user context for logging - 'x-customer-id': user.customer_id, - 'x-customer-name': user.customer_name || '', - 'x-user-id': user.user_id || '', - 'x-username': user.username || '', - 'x-customer-tier': user.customer_tier || '', }; this.logger.info('Proxying request to storage-explorer', { @@ -132,12 +126,6 @@ export class StorageExplorerService { } const headers: Record = { - // Forward user context for logging - 'x-customer-id': user.customer_id, - 'x-customer-name': user.customer_name || '', - 'x-user-id': user.user_id || '', - 'x-username': user.username || '', - 'x-customer-tier': user.customer_tier || '', // Let axios set Content-Type for multipart/form-data with boundary ...formData.getHeaders?.(), }; From 3ab2f8f27d0bca1388b910342e4bb5fd0f556148 Mon Sep 17 00:00:00 2001 From: viniciusgadea Date: Wed, 14 Jan 2026 08:51:45 -0300 Subject: [PATCH 17/18] CHORE: remove linkFileToTable endpoint from Storage Explorer controller --- .../storage-explorer.controller.ts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/modules/storage-explorer/storage-explorer.controller.ts b/src/modules/storage-explorer/storage-explorer.controller.ts index fd2e4d9..ef6a071 100644 --- a/src/modules/storage-explorer/storage-explorer.controller.ts +++ b/src/modules/storage-explorer/storage-explorer.controller.ts @@ -332,21 +332,6 @@ export class StorageExplorerController { ); } - @ApiOperation({ summary: 'Link an existing file in storage to a table' }) - @Post('storage/link-to-table') - @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.WRITE) - async linkFileToTable( - @Body() body: any, - @User() user: RequestUser, - ) { - return this.storageExplorerService.proxy( - 'POST', - '/storage/link-to-table', - user, - body, - ); - } - @ApiOperation({ summary: 'Get detailed file metadata' }) @Get('storage/metadata') @RequireAllPermissions(PERMISSIONS_GROUPS.STORAGE_EXPLORER.permissions.READ) From 4246c7495efd69e2f1119ad1143313b726992754 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 20 Jan 2026 17:52:34 -0300 Subject: [PATCH 18/18] UPDATE: updating /me to get api key --- docsfera.json | 516 ++++++++++++++-------------- src/modules/auth/auth.controller.ts | 25 +- src/modules/auth/auth.module.ts | 3 +- 3 files changed, 275 insertions(+), 269 deletions(-) diff --git a/docsfera.json b/docsfera.json index 8942a26..7d13f03 100644 --- a/docsfera.json +++ b/docsfera.json @@ -715,6 +715,158 @@ ] } }, + "/api-key": { + "post": { + "operationId": "ApiKeyController_create", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyDto" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyResponseDto" + } + } + } + }, + "default": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateApiKeyResponseDto" + } + } + } + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + }, + "get": { + "operationId": "ApiKeyController_findAll", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiKeyBaseResponseDto" + } + } + } + } + }, + "default": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ApiKeyBaseResponseDto" + } + } + } + } + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, + "/api-key/{id}": { + "delete": { + "operationId": "ApiKeyController_remove", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "ApiKey" + ], + "security": [ + { + "access-token": [] + } + ] + } + }, "/connections": { "post": { "operationId": "ConnectionController_createConnection", @@ -6584,158 +6736,6 @@ ] } }, - "/api-key": { - "post": { - "operationId": "ApiKeyController_create", - "parameters": [ - { - "name": "dadosfera-lang", - "in": "header", - "required": false, - "schema": { - "enum": [ - "pt-br", - "en-us" - ], - "type": "string" - } - } - ], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateApiKeyDto" - } - } - } - }, - "responses": { - "201": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateApiKeyResponseDto" - } - } - } - }, - "default": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateApiKeyResponseDto" - } - } - } - } - }, - "tags": [ - "ApiKey" - ], - "security": [ - { - "access-token": [] - } - ] - }, - "get": { - "operationId": "ApiKeyController_findAll", - "parameters": [ - { - "name": "dadosfera-lang", - "in": "header", - "required": false, - "schema": { - "enum": [ - "pt-br", - "en-us" - ], - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ApiKeyBaseResponseDto" - } - } - } - } - }, - "default": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ApiKeyBaseResponseDto" - } - } - } - } - } - }, - "tags": [ - "ApiKey" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, - "/api-key/{id}": { - "delete": { - "operationId": "ApiKeyController_remove", - "parameters": [ - { - "name": "dadosfera-lang", - "in": "header", - "required": false, - "schema": { - "enum": [ - "pt-br", - "en-us" - ], - "type": "string" - } - }, - { - "name": "id", - "required": true, - "in": "path", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "" - } - }, - "tags": [ - "ApiKey" - ], - "security": [ - { - "access-token": [] - } - ] - } - }, "/identity-providers": { "post": { "operationId": "IdentityProviderController_addIdentityProvider", @@ -7851,6 +7851,104 @@ "accessToken" ] }, + "CreateApiKeyDto": { + "type": "object", + "properties": { + "permissions": { + "description": "Array of permission IDs", + "type": "array", + "items": { + "type": "number" + } + } + }, + "required": [ + "permissions" + ] + }, + "PermissionDto": { + "type": "object", + "properties": { + "id": { + "type": "number" + }, + "name": { + "type": "string" + } + }, + "required": [ + "id", + "name" + ] + }, + "CreateApiKeyResponseDto": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "key_mask": { + "type": "string" + }, + "permissions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PermissionDto" + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "type": "string" + }, + "key": { + "type": "string" + } + }, + "required": [ + "id", + "key_mask", + "permissions", + "created_at", + "created_by", + "key" + ] + }, + "ApiKeyBaseResponseDto": { + "type": "object", + "properties": { + "id": { + "type": "string", + "format": "uuid" + }, + "key_mask": { + "type": "string" + }, + "permissions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/PermissionDto" + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "type": "string" + } + }, + "required": [ + "id", + "key_mask", + "permissions", + "created_at", + "created_by" + ] + }, "CreateConnectionDto": { "type": "object", "properties": { @@ -8074,37 +8172,6 @@ "message" ] }, - "PermissionDto": { - "type": "object", - "properties": { - "id": { - "type": "number" - }, - "seqid": { - "type": "number" - }, - "name": { - "type": "string" - }, - "claim": { - "type": "string" - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - }, - "required": [ - "id", - "seqid", - "name", - "claim", - "createdAt", - "updatedAt" - ] - }, "Customer": { "type": "object", "properties": { @@ -10588,89 +10655,6 @@ "publicKey" ] }, - "CreateApiKeyDto": { - "type": "object", - "properties": { - "permissions": { - "description": "Array of permission IDs", - "type": "array", - "items": { - "type": "number" - } - } - }, - "required": [ - "permissions" - ] - }, - "CreateApiKeyResponseDto": { - "type": "object", - "properties": { - "id": { - "type": "string", - "format": "uuid" - }, - "key_mask": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PermissionDto" - } - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "created_by": { - "type": "string" - }, - "key": { - "type": "string" - } - }, - "required": [ - "id", - "key_mask", - "permissions", - "created_at", - "created_by", - "key" - ] - }, - "ApiKeyBaseResponseDto": { - "type": "object", - "properties": { - "id": { - "type": "string", - "format": "uuid" - }, - "key_mask": { - "type": "string" - }, - "permissions": { - "type": "array", - "items": { - "$ref": "#/components/schemas/PermissionDto" - } - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "created_by": { - "type": "string" - } - }, - "required": [ - "id", - "key_mask", - "permissions", - "created_at", - "created_by" - ] - }, "CreateIdentityProvider": { "type": "object", "properties": { diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 34dfd06..7c3e073 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -55,6 +55,7 @@ import jwt, { JwtPayload } from 'jsonwebtoken'; import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; +import { ApiKeyService } from 'src/modules/api-key/api-key.service'; type CookiesValues = { accessToken?: string; @@ -74,6 +75,7 @@ export class AuthController { @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, private authClient: AuthClientService, + private apiKeyService: ApiKeyService, ) { this.logger = dadosferaLogger.logger; @@ -477,10 +479,29 @@ export class AuthController { async getMe(@Req() req: Request, @Res() res: Response) { this.logger.info('GET /auth/me ') + // Check for API key header first + const apiKey = req.get('X-Api-key'); + if (apiKey) { + this.logger.info('Authenticating via X-Api-key header'); + const { api_key } = await this.apiKeyService.get(apiKey); + + const userDto = { + id: api_key.user_id, + name: api_key.username, + customer: { + id: api_key.customer_id, + name: api_key.customer_name, + tier: api_key.customer_tier, + } + }; + + return res.status(200).json(userDto); + } + // Get token and headers const accessToken = req.cookies['ddf-auth']; - const refreshToken = req.cookies['ddf-refresh-auth']; - const userId = req.cookies['ddf-user-id']; + const refreshToken = req.cookies['ddf-refresh-auth']; + const userId = req.cookies['ddf-user-id']; const resourceHost = req.headers["host"] const hasUserSession = Boolean(accessToken) && Boolean(userId); diff --git a/src/modules/auth/auth.module.ts b/src/modules/auth/auth.module.ts index 24e7623..47b59cc 100644 --- a/src/modules/auth/auth.module.ts +++ b/src/modules/auth/auth.module.ts @@ -8,10 +8,11 @@ import { AuthClientService } from './auth.service'; import { DucClient } from '../duc/client.config'; import { GoogleLoginStrategy } from './passport-strategies/google-strategy'; import { getOauthSecrets } from 'src/utils/OauthSecrets'; +import { ApiKeyModule } from '../api-key/api-key.module'; const client = new DucClient(); @Module({ - imports: [ClientsModule.register([client.providerOptions])], + imports: [ClientsModule.register([client.providerOptions]), ApiKeyModule], controllers: [AuthController], providers: [ AuthClientService,