Compare commits

...
Author SHA1 Message Date
Claude e0bfe03898 DOCS: Map maestro-pifactory integration points for migration
Add comprehensive documentation mapping all integration points between
maestro and pi-factory services to facilitate removing the dependency.

Includes:
- 3 gRPC client configurations using PIFACTORY_URL
- 36+ gRPC method calls across Catalog, PipelineV2, and legacy Pipeline services
- 44 REST endpoints that proxy to pi-factory
- Configuration files and environment variables
- Proto package dependencies (protospack and protospack-v2)
- Migration strategy options and recommendations
- Comparison with in-factory migration scope
2025-12-17 18:21:38 +00:00
Claude 177ec6368c DOCS: Map maestro-infactory integration points for migration
Add comprehensive documentation mapping all integration points between
maestro and in-factory services to facilitate removing the dependency.

Includes:
- All 6 gRPC client configurations using INFACTORY_URL
- 27+ gRPC method calls across 6 services
- 25 REST endpoints that proxy to in-factory
- Configuration files and environment variables
- Proto package dependencies
- Migration strategy options and recommendations
2025-12-17 17:09:50 +00:00
Marcos Rodrigues Silva e7f410831f Merge pull request #395 from dadosfera/beta
Beta
2025-12-01 17:47:24 -03:00
Marcos Rodrigues Silva cd2c53b5c5 Merge pull request #398 from dadosfera/feat/user-properties
FIX: user response
2025-12-01 17:36:56 -03:00
Marcos Rodrigues adf1b3b97e FIX: user response 2025-12-01 17:30:37 -03:00
Marcos Rodrigues Silva d84b5e184b Merge pull request #397 from dadosfera/feat/user-properties
FEAT: include more user info in /me
2025-12-01 12:36:16 -03:00
Marcos Rodrigues 6d608a0457 FEAT: include more user info in /me 2025-12-01 12:26:33 -03:00
Marcos Rodrigues Silva b8be2c7803 Merge pull request #396 from dadosfera/fix/api-key-catalog-roles
FIX: api key catalog roles
2025-12-01 09:34:21 -03:00
Marcos Rodrigues 24fce721e3 FIX: api key catalog roles 2025-12-01 09:32:26 -03:00
Marcos Rodrigues Silva 23a9a27db1 Merge pull request #394 from dadosfera/fix/private-front-cors
FIX: 404 in  get public key
2025-11-28 16:44:54 -03:00
Marcos Rodrigues 0f5ed50af9 FIX: 404 in get public key 2025-11-28 16:37:38 -03:00
Marcos Rodrigues Silva 5f8f6a64ab Merge pull request #393 from dadosfera/fix/private-front-cors
FIX: include private frontend url in cors origins list
2025-11-28 10:53:45 -03:00
Marcos Rodrigues 8800ac2736 FIX: include private frontend url in cors origins list 2025-11-28 10:50:31 -03:00
Marcos Rodrigues Silva bdb82c2ce4 Merge pull request #392 from dadosfera/fix/authenticated-track-events
Fix/authenticated track events
2025-11-27 09:45:46 -03:00
Marcos Rodrigues 6d9ecc3568 FIX: mixpanel events 2025-11-27 09:43:43 -03:00
Marcos Rodrigues Silva 7764447adc Merge pull request #391 from dadosfera/fix/private-ingress-annotations
FIX: ci to oracle github worker
2025-11-25 11:37:29 -03:00
Marcos Rodrigues 576fdecf89 FIX: ci to oracle github worker 2025-11-25 11:27:04 -03:00
Marcos Rodrigues Silva 3de1e90fa8 Merge pull request #390 from dadosfera/fix/private-ingress-annotations
Fix/private ingress annotations
2025-11-25 11:12:58 -03:00
Marcos Rodrigues 5e90950660 FIX: restore whitelist annotation 2025-11-25 10:59:06 -03:00
Marcos Rodrigues 3ed26e648f FIX: ci 2025-11-25 10:48:06 -03:00
Marcos Rodrigues Silva fff3523152 Merge pull request #389 from dadosfera/fix/private-ingress-annotations
FIX: private ingress annotations
2025-11-25 10:38:52 -03:00
Marcos Rodrigues 19e4daeea4 FIX: private ingress annotations 2025-11-25 10:35:08 -03:00
Marcos Rodrigues Silva 8624d3f016 Merge pull request #388 from dadosfera/fix/reset-theme
Fix/reset theme
2025-11-21 11:01:48 -03:00
Marcos Rodrigues 4f4da5bebe FIX: reset theme 2025-11-21 10:51:31 -03:00
Marcos Rodrigues f00d2bf41d FIX: reset cstumer theme 2025-11-21 09:55:39 -03:00
Marcos Rodrigues Silva dc1d1400d8 Merge pull request #387 from dadosfera/feat/lineage
FIX: change wget to curl
2025-11-13 18:19:20 -03:00
Marcos Rodrigues d4451153a3 FIX: change wget to curl 2025-11-13 18:14:47 -03:00
Marcos Rodrigues Silva 06ba759ea3 Merge pull request #386 from dadosfera/feat/lineage
Feat/lineage
2025-11-13 17:58:33 -03:00
Marcos Rodrigues dd0d08ad6e FEAT: add permissions to lineage module 2025-11-13 17:48:55 -03:00
Marcos Rodrigues ec81082877 FIX: resolve cors from localhost 2025-11-13 17:48:38 -03:00
Marcos Rodrigues Silva 7f58dc7090 Merge pull request #385 from dadosfera/hotfix/import-files
FIX: permissions
2025-10-22 18:35:35 -03:00
Marcos Rodrigues 43aa379c06 FIX: permissions 2025-10-22 18:29:43 -03:00
Marcos Rodrigues Silva c284f8753c Merge pull request #384 from dadosfera/hotfix/import-files
FIX: pipelines v2 permissions
2025-10-22 17:48:26 -03:00
Marcos Rodrigues 3a8f2495c4 FIX: pipelines v2 permissions 2025-10-20 18:13:54 -03:00
Marcos Rodrigues Silva af248716ef Merge pull request #383 from dadosfera/beta
Beta
2025-10-16 18:15:08 -03:00
Marcos Rodrigues Silva a57ad41ad4 Merge pull request #382 from dadosfera/features/roles
Features/roles
2025-10-16 17:21:37 -03:00
Marcos Rodrigues ffddceec3b FEAT: roles 2025-10-16 17:17:01 -03:00
Marcos Rodrigues Silva b04d5bb402 Merge pull request #381 from dadosfera/feature/tableau-dash
UPDATE: default roles and user
2025-09-23 18:10:35 -03:00
Marcos Rodrigues 53df3caf3f UPDATE: default roles and user 2025-09-23 18:09:25 -03:00
Marcos Rodrigues Silva ab2e35b54f Merge pull request #380 from dadosfera/beta
Beta
2025-09-17 18:11:05 -03:00
Marcos Rodrigues Silva bfa77d8f7b Merge pull request #379 from dadosfera/feature/auth-cookie
Feature/auth cookie
2025-09-16 17:40:10 -03:00
marcos-silva-rodrigues 5002d147ad UPDATE: add all dadosfera domains 2025-09-16 17:33:19 -03:00
Marcos Rodrigues Silva cd55dc0dc4 Merge pull request #378 from dadosfera/hotfix/uptime-ip
UPDATE: uptime internal ip
2025-09-15 18:04:14 -03:00
marcos-silva-rodrigues f1d56e4c2c UPDATE: uptime internal ip 2025-09-15 10:34:36 -03:00
Marcos Rodrigues Silva bc49f79cb3 Merge pull request #373 from dadosfera/hotfix/oracle
Network policies
2025-09-09 18:08:28 -03:00
marcos-silva-rodrigues 00163ad894 UPDATE: network policies 2025-09-09 11:17:45 -03:00
Marcos Rodrigues Silva a2fbeb97cc Merge pull request #377 from dadosfera/feature/auth-cookie
UPDATE: test cors domain
2025-09-08 15:25:05 -03:00
marcos-silva-rodrigues 288a796f46 UPDATE: test cors domain 2025-09-08 15:22:26 -03:00
Marcos Rodrigues Silva b5a1e93770 Merge pull request #376 from dadosfera/feature/auth-cookie
UPDATE: cookie path
2025-09-08 10:32:32 -03:00
marcos-silva-rodrigues 1ee49ceab8 UPDATE: cookie path 2025-09-08 10:30:56 -03:00
Marcos Rodrigues Silva 6eaf9cf6d0 Merge pull request #375 from dadosfera/feature/auth-cookie
UDATE: test cookie domain dadosfera.ai
2025-09-08 09:32:21 -03:00
marcos-silva-rodrigues 075ca747df FIX: ghost commit 2025-09-08 09:26:47 -03:00
marcos-silva-rodrigues 9ead4588c1 UDATE: test cookie domain dadosfera.ai 2025-09-08 09:25:14 -03:00
Marcos Rodrigues Silva 92b64bb362 Merge pull request #374 from dadosfera/hotfix/oracle
Hotfix/oracle
2025-09-05 10:25:38 -03:00
marcos-silva-rodrigues cb8798d871 FIX: enable restricted ips 2025-09-05 10:21:11 -03:00
marcos-silva-rodrigues be70f4da09 FIX: disabled network policies 2025-09-04 10:02:20 -03:00
Marcos Rodrigues Silva b436d7de7a Merge pull request #372 from dadosfera/hotfix/oracle
Hotfix/oracle
2025-09-03 16:49:31 -03:00
Marcos Rodrigues Silva 6780167f2b Merge pull request #371 from dadosfera/hotfix/oracle
FIX: infinite loading when login is incorrect
2025-09-03 16:31:28 -03:00
marcos-silva-rodrigues 2a3ab4228f FIX: infinite loading when login is incorrect 2025-09-03 16:25:46 -03:00
Marcos Rodrigues Silva caeaf62a9d Merge pull request #370 from dadosfera/hotfix/oracle
Hotfix/oracle
2025-09-03 12:21:17 -03:00
marcos-silva-rodrigues 8e605aa361 FIX: affinity and tls config local 2025-09-03 12:19:30 -03:00
Marcos Rodrigues Silva fa4267f54a Merge pull request #369 from dadosfera/hotfix/oracle
UPDATE: oracle vpn
2025-08-29 10:16:45 -03:00
marcos-silva-rodrigues 47d8ca2760 UPDATE: oracle vpn 2025-08-29 10:12:58 -03:00
Marcos Rodrigues Silva f3c51e5328 Merge pull request #368 from dadosfera/beta
UPDATE: fixed cloud oracle
2025-08-28 18:54:02 -03:00
marcos-silva-rodrigues dcf70fdaae UPDATE: fixed cloud oracle 2025-08-28 18:40:43 -03:00
Marcos Rodrigues Silva 70f663374d Merge pull request #366 from dadosfera/beta
MIgração para oracle
2025-08-28 18:32:59 -03:00
Marcos Rodrigues Silva 38f2317bb6 Merge pull request #367 from dadosfera/chore/oracle
UPDATE: affinity and log
2025-08-28 17:05:32 -03:00
marcos-silva-rodrigues 31e0ca6c91 UPDATE: affinity and log 2025-08-28 16:52:52 -03:00
Marcos Rodrigues Silva dc063cbf3f Merge pull request #365 from dadosfera/chore/oracle
Chore/oracle
2025-08-27 17:45:34 -03:00
marcos-silva-rodrigues a4c82ae4a8 UPDATE:migration to oracle 2025-08-27 17:38:26 -03:00
Marcos Rodrigues Silva 11a65e11d1 Merge pull request #364 from dadosfera/chore/oracle
UPDATE: redis host
2025-08-27 14:15:27 -03:00
marcos-silva-rodrigues 73d47f0ff5 UPDATE: merge 2025-08-27 14:13:17 -03:00
marcos-silva-rodrigues d36e532c8e UPDATE: redis host 2025-08-27 13:58:31 -03:00
Marcos Rodrigues Silva b2a24e3a49 Merge pull request #363 from dadosfera/feature/auth-session
UPDATE: test subdomain
2025-08-14 17:51:36 -03:00
Marcos Rodrigues Silva 381a401ebb Merge pull request #362 from dadosfera/feature/auth-session
CI: add cookie secret
2025-08-14 17:27:09 -03:00
Marcos Rodrigues Silva 1b6846532f Merge pull request #361 from dadosfera/feature/auth-session
CHORE: update nginx ingress annotations
2025-08-14 16:07:31 -03:00
Marcos Rodrigues Silva 878c9cf450 Merge pull request #360 from dadosfera/feature/auth-session
FIX: set cookies if exists tokens
2025-08-14 15:39:57 -03:00
Marcos Rodrigues Silva df852449b2 Merge pull request #359 from dadosfera/feature/auth-session
Feature/auth session
2025-08-14 15:28:23 -03:00
Marcos Rodrigues Silva 9b168612a4 Merge pull request #358 from dadosfera/beta
Beta
2025-08-08 11:23:02 -03:00
34 changed files with 1725 additions and 219 deletions
+1 -1
View File
@@ -146,7 +146,7 @@ jobs:
needs: [extract_environment, semantic_release, build_ecr_image]
uses: ./.github/workflows/k8s-deploy.yml
with:
cloud: ${{ (needs.extract_environment.outputs.environment == 'prd' && 'azure' ) || 'oracle' }}
cloud: 'oracle'
environment: ${{ needs.extract_environment.outputs.environment }}
image: ${{ needs.semantic_release.outputs.new_release_version }}
secrets: inherit
+3 -3
View File
@@ -56,9 +56,9 @@ jobs:
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
mv helmfile /usr/local/bin/
sudo mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
@@ -105,7 +105,7 @@ jobs:
- name: Install Helmfile
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
curl -fsSLO https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
helmfile --version
+482
View File
@@ -0,0 +1,482 @@
# Maestro ↔ In-Factory Migration Map
This document maps all integration points between Maestro and In-Factory that need to be addressed to remove the dependency.
## Architecture Overview
```mermaid
graph TD;
Frontend<-->Maestro;
Maestro<-->DUC;
Maestro<-->pi-factory;
Maestro<-->in-factory;
```
Maestro connects to **3 external microservices**:
- **DUC** (`DUC_URL`) - User management, authentication, permissions
- **PI-Factory** (`PIFACTORY_URL`) - Pipelines, Catalog services
- **IN-Factory** (`INFACTORY_URL`) - Connections, Connectors, Inputs, Network Config, Transformations
---
## Summary Table
| Category | Count | Impact Level |
|----------|-------|--------------|
| gRPC Client Configurations | 6 | HIGH |
| NestJS Modules | 6 | HIGH |
| REST Controllers/Endpoints | 6 | HIGH |
| Service Classes | 6 | HIGH |
| Configuration Files | 5 | MEDIUM |
| Proto Package Dependencies | 2 | HIGH |
| Environment Variables | 2 | LOW |
---
## 1. gRPC CLIENT CONFIGURATIONS (Files to Migrate)
These files configure gRPC connections to In-Factory services:
### 1.1 Connection Client (`INFACTORY_URL`)
**File:** `src/modules/connection/client.config.ts`
```typescript
// Lines 10-11, 18
process.env.INFACTORY_URL.startsWith('in-factory:')
process.env.INFACTORY_URL.includes('0.0.0.0')
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `ConnectionManager.ProtoPackages.WritePackage`
- `ConnectionManager.ProtoPackages.ReadPackage`
### 1.2 Connector Client (`INFACTORY_URL`)
**File:** `src/modules/connector/client.config.ts`
```typescript
// Lines 10-11, 18
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `ConnectorManager.ProtoPackages.WritePackage`
- `ConnectorManager.ProtoPackages.ReadPackage`
### 1.3 Inputs Client (`INFACTORY_URL`)
**File:** `src/modules/inputs/inputs-client.config.ts`
```typescript
// Lines 10-11, 17
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `Input.ProtoPackages.WritePackage`
- `Input.ProtoPackages.ReadPackage`
### 1.4 Network Config Client (`INFACTORY_URL`)
**File:** `src/modules/network-config/network-config-client.config.ts`
```typescript
// Lines 10-11, 19
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `NetworkConfig.ProtoPackages.WritePackage`
- `NetworkConfig.ProtoPackages.ReadPackage`
### 1.5 Connection Test Client (`INFACTORY_URL`)
**File:** `src/modules/connection-test/connection-test-client.config.ts`
```typescript
// Lines 10-11, 17
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `ConnectionTest.ProtoPackages.ReadPackage`
### 1.6 Transformations Client (`INFACTORY_URL`)
**File:** `src/modules/transformations/transformations-client.ts`
```typescript
// Lines 10-11, 18
url: process.env.INFACTORY_URL
```
**Proto Services Used:**
- `Transformation.ProtoPackages.WritePackage`
---
## 2. SERVICE CLASSES (Business Logic to Migrate)
### 2.1 Connection Client Service
**File:** `src/modules/connection/client.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `CreateConnection()` | ConnectionManagerWriteServices | Write |
| `UpdateConnection()` | ConnectionManagerWriteServices | Write |
| `DeleteConnection()` | ConnectionManagerWriteServices | Write |
| `GetConnectionDetails()` | ConnectionManagerReadServices | Read |
| `GetAllConnections()` | ConnectionManagerReadServices | Read |
| `GetConnectorAvailableConnectionsByCustomer()` | ConnectionManagerReadServices | Read |
| `ValidatePlatformConnections()` | ConnectionManagerReadServices | Read |
**Key Methods:**
- `createConnection()`
- `updateConnection()`
- `deleteConnection()`
- `getConnections()`
- `getConnectionDetails()`
- `validatePlatformConnections()`
### 2.2 Connector Client Service
**File:** `src/modules/connector/client.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `RegisterConnector()` | ConnectorManagerWriteServices | Write |
| `UploadFile()` | ConnectorManagerWriteServices | Write |
| `RegisterMultipleConnectorsWithoutImage()` | ConnectorManagerWriteServices | Write |
| `UpdateConnectorByID()` | ConnectorManagerWriteServices | Write |
| `DeleteConnectorById()` | ConnectorManagerWriteServices | Write |
| `GetConnectors()` | ConnectorManagerReadServices | Read |
### 2.3 Inputs Service
**File:** `src/modules/inputs/inputs.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `CreateInput()` | InputWriteService | Write |
| `UpdateInput()` | InputWriteService | Write |
| `DeleteInput()` | InputWriteService | Write |
| `GetAvailableEntities()` | InputReadService | Read |
### 2.4 Network Config Service
**File:** `src/modules/network-config/network-config.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `NetworkConfigCreate()` | NetworkConfigWriteService | Write |
| `NetworkConfigUpdate()` | NetworkConfigWriteService | Write |
| `NetworkConfigDelete()` | NetworkConfigWriteService | Write |
| `NetworkConfigFindAll()` | NetworkConfigReadService | Read |
| `NetworkConfigFindOneById()` | NetworkConfigReadService | Read |
### 2.5 Connection Test Service
**File:** `src/modules/connection-test/connection-test.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `TestConnection()` | ConnectionTestReadService | Read |
### 2.6 Transformations Service
**File:** `src/modules/transformations/client.service.ts`
**gRPC Methods Called:**
| Method | Service | Direction |
|--------|---------|-----------|
| `CreateTransformation()` | TransformationWriteService | Write |
| `UpdateTransformation()` | TransformationWriteService | Write |
| `DeleteTransformation()` | TransformationWriteService | Write |
| Various read operations | TransformationReadService | Read |
---
## 3. NESTJS MODULES (Module Registration)
These modules register the gRPC clients and export services:
| Module | File | Imports | Exports |
|--------|------|---------|---------|
| ConnectionModule | `src/modules/connection/connection.module.ts` | ConnectionClientConfiguration | ConnectionClientService |
| ConnectorModule | `src/modules/connector/connector.module.ts` | ConnectorClientConfiguration | ConnectorClientService |
| InputsModule | `src/modules/inputs/inputs.module.ts` | InputsGrpcClient | InputsService |
| NetworkConfigModule | `src/modules/network-config/network-config.module.ts` | NetworkConfigGrpcClient | NetworkConfigService |
| ConnectionTestModule | `src/modules/connection-test/connection-test.module.ts` | ConnectionTestClientConfiguration | ConnectionTestService |
| TransformationsModule | `src/modules/transformations/transformations.module.ts` | TransformationsClientConfiguration | TransformationsService |
**App Module Registration:** `src/app.module.ts` (Lines 15-21, 50-60)
---
## 4. REST CONTROLLERS (API Endpoints to Migrate)
### 4.1 Connection Controller
**File:** `src/modules/connection/connection.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/connections` | Create a new connection |
| PUT | `/connections/:id` | Update an existing connection |
| DELETE | `/connections/:id` | Delete a connection |
| GET | `/connections` | Get all connections |
| GET | `/connections/:id` | Get connection details |
| GET | `/connections/available/:connector_id` | Get available connections by connector |
### 4.2 Connector Controller
**File:** `src/modules/connector/connector.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/connectors` | Register a new connector |
| POST | `/connectors/uploads` | Upload multiple connector files |
| GET | `/connectors` | List all connectors |
| PUT | `/connectors/:id` | Update a connector |
| DELETE | `/connectors/:id` | Delete a connector |
### 4.3 Inputs Controller
**File:** `src/modules/inputs/inputs.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| GET | `/inputs/available-entities/:plugin` | Get available entities for a plugin |
| POST | `/inputs` | Create an input |
| PATCH | `/inputs` | Update an input |
| DELETE | `/inputs/:id` | Delete an input |
### 4.4 Network Config Controller
**File:** `src/modules/network-config/network-config.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| GET | `/network-configs` | Get all network configurations |
| GET | `/network-configs/:id` | Get network config by ID |
| POST | `/network-configs` | Create network configuration |
| PUT | `/network-configs/:id` | Update network configuration |
| DELETE | `/network-configs/:id` | Delete network configuration |
### 4.5 Connection Test Controller
**File:** `src/modules/connection-test/connection-test.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/connection-test` | Test a connection |
### 4.6 Transformations Controller
**File:** `src/modules/transformations/transformations.controller.ts`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/transformations` | Create transformation |
| GET | `/transformations` | Get transformations |
| PUT | `/transformations/:id` | Update transformation |
| DELETE | `/transformations/:id` | Delete transformation |
---
## 5. CONFIGURATION FILES
### 5.1 Helm Chart Values (Production)
**File:** `deploy/helm-chart/values.yaml`
```yaml
# Lines 32, 42
maestro:
in_factory_url: in-factory.dadosfera.ai
tr_factory_url: in-factory.dadosfera.ai # Transformation factory also uses in-factory
```
### 5.2 Helm Chart Values (Staging)
**File:** `deploy/helm-chart/values-stg.yaml`
```yaml
# Lines 5-6
maestro:
in_factory_url: in-factory.stg.dadosfera.ai
tr_factory_url: in-factory.stg.dadosfera.ai
```
### 5.3 Helmfiles (Production)
**File:** `deploy/helmfiles/prd.yaml`
```yaml
# Lines 15-18, 39-42 (for both maestro and maestro-unimed releases)
maestro.in_factory_url: in-factory.dadosfera.ai
maestro.tr_factory_url: in-factory.dadosfera.ai
```
### 5.4 Deployment Template
**File:** `deploy/helm-chart/templates/deployment.yaml`
```yaml
# Lines 71-72
- name: INFACTORY_URL
value: {{ .Values.maestro.in_factory_url }}
```
### 5.5 Environment Type Definition
**File:** `environment.d.ts`
```typescript
// Line 9
INFACTORY_URL: string;
```
---
## 6. PROTO PACKAGE DEPENDENCIES
### 6.1 Package.json
**File:** `package.json`
```json
{
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.14"
}
```
### 6.2 Proto Imports from `@dadosfera/protospack-v2` (In-Factory related)
| Import Path | Used In |
|-------------|---------|
| `ConnectionManager` | connection/client.config.ts, connection/client.service.ts |
| `ConnectionManager/interfaces/messages` | connection/client.service.ts, connection/dtos/connection.ts |
| `ConnectionManager/interfaces/entities` | connection/dtos/connection.ts |
| `ConnectorManager` | connector/client.config.ts, connector/client.service.ts |
| `Input` | inputs/inputs-client.config.ts, inputs/inputs.service.ts |
| `Input/interfaces/messages` | inputs/inputs.service.ts |
| `Input/interfaces/entities` | inputs/inputs.controller.ts, inputs/inputs.service.ts |
| `NetworkConfig` | network-config/network-config-client.config.ts, network-config/network-config.service.ts |
| `NetworkConfig/interfaces/entities` | network-config/dto/network-config.ts |
| `ConnectionTest` | connection-test/connection-test-client.config.ts, connection-test/connection-test.service.ts |
| `Transformation` | transformations/transformations-client.ts, transformations/client.service.ts |
---
## 7. MIGRATION STRATEGY OPTIONS
### Option A: Move In-Factory functionality INTO Maestro
**Pros:**
- Single service to maintain
- No network latency for these operations
- Simpler deployment
**Cons:**
- Increases Maestro's responsibility/complexity
- Requires database access from Maestro
- May require significant refactoring
**Files to create/migrate:**
1. Database models for connections, connectors, inputs, network-configs, transformations
2. Repository layer for database operations
3. Convert gRPC services to internal services
4. Remove all gRPC client configurations
### Option B: Create REST API wrapper in In-Factory
**Pros:**
- Minimal changes to Maestro
- Can migrate incrementally
**Cons:**
- Still maintains dependency
- Additional REST→gRPC translation layer
### Option C: Direct database access from Maestro
**Pros:**
- Removes runtime dependency
- Better performance
**Cons:**
- Shared database coupling
- Complex migration
---
## 8. FILES TO MODIFY/DELETE (Summary)
### High Priority - Core Integration Files
```
src/modules/connection/client.config.ts → DELETE or REPLACE
src/modules/connection/client.service.ts → REPLACE with local implementation
src/modules/connector/client.config.ts → DELETE or REPLACE
src/modules/connector/client.service.ts → REPLACE with local implementation
src/modules/inputs/inputs-client.config.ts → DELETE or REPLACE
src/modules/inputs/inputs.service.ts → REPLACE with local implementation
src/modules/network-config/network-config-client.config.ts → DELETE or REPLACE
src/modules/network-config/network-config.service.ts → REPLACE with local implementation
src/modules/connection-test/connection-test-client.config.ts → DELETE or REPLACE
src/modules/connection-test/connection-test.service.ts → REPLACE with local implementation
src/modules/transformations/transformations-client.ts → DELETE or REPLACE
src/modules/transformations/client.service.ts → REPLACE with local implementation
```
### Medium Priority - Module Registration
```
src/modules/connection/connection.module.ts → UPDATE imports
src/modules/connector/connector.module.ts → UPDATE imports
src/modules/inputs/inputs.module.ts → UPDATE imports
src/modules/network-config/network-config.module.ts → UPDATE imports
src/modules/connection-test/connection-test.module.ts → UPDATE imports
src/modules/transformations/transformations.module.ts → UPDATE imports
src/app.module.ts → UPDATE if module structure changes
```
### Low Priority - Configuration
```
deploy/helm-chart/values.yaml → REMOVE in_factory_url, tr_factory_url
deploy/helm-chart/values-stg.yaml → REMOVE in_factory_url, tr_factory_url
deploy/helmfiles/prd.yaml → REMOVE in_factory_url references
deploy/helm-chart/templates/deployment.yaml → REMOVE INFACTORY_URL env var
environment.d.ts → REMOVE INFACTORY_URL type
README.md → UPDATE architecture diagram
```
---
## 9. DEPENDENCY COUNT BY MODULE
| Module | Files | gRPC Calls | REST Endpoints |
|--------|-------|------------|----------------|
| Connection | 4 | 7 | 6 |
| Connector | 3 | 6 | 5 |
| Inputs | 3 | 4 | 4 |
| Network Config | 3 | 5 | 5 |
| Connection Test | 3 | 1 | 1 |
| Transformations | 4 | 4+ | 4 |
| **TOTAL** | **20** | **27+** | **25** |
---
## 10. DATA MODELS (Proto Messages Used)
### Connection Manager
- `CreateConnectionRequest` / `CreateConnectionResponse`
- `UpdateConnectionRequest` / `UpdateConnectionResponse`
- `DeleteConnectionRequest` / `DeleteConnectionResponse`
- `GetConnectionDetailsRequest` / `GetConnectionDetailsResponse`
- `GetAllConnectionsRequest` / `GetAllConnectionsResponse`
- `Connection` (entity)
- `ConnectionCredential` (entity)
### Connector Manager
- `RegisterConnectorRequest` / `RegisterConnectorResponse`
- `UploadFileRequest` / `UploadFileResponse`
- `GetConnectorsRequest` / `GetConnectorsResponse`
- `Connector` (entity)
### Input
- `CreateInputRequest` / `CreateInputResponse`
- `UpdateInputRequest` / `UpdateInputResponse`
- `DeleteInputRequest` / `DeleteInputResponse`
- `GetAvailableEntitiesRequest` / `GetAvailableEntitiesResponse`
- `Info` (entity)
### Network Config
- `NetworkConfigCreateRequest` / `NetworkConfigCreateResponse`
- `NetworkConfigUpdateRequest` / `NetworkConfigUpdateResponse`
- `NetworkConfigDeleteRequest` / `NetworkConfigDeleteResponse`
- `NetworkConfigFindAllRequest` / `NetworkConfigFindAllResponse`
- `NetworkConfig` (entity)
### Connection Test
- `TestConnectionRequest` / `TestConnectionResponse`
### Transformation
- `CreateTransformationRequest` / `CreateTransformationResponse`
- `UpdateTransformationRequest` / `UpdateTransformationResponse`
- `DeleteTransformationRequest` / `DeleteTransformationResponse`
---
## NEXT STEPS
1. **Decide on migration strategy** (Option A, B, or C)
2. **Prioritize modules** - Recommend starting with Connection Test (smallest), then Inputs, Network Config, Transformations, Connection, Connector (largest)
3. **Create database schema** if moving to Option A
4. **Implement local services** one module at a time
5. **Update tests** for each migrated module
6. **Update deployment configs** to remove INFACTORY_URL
7. **Coordinate with In-Factory team** for data migration
+320
View File
@@ -0,0 +1,320 @@
# In-Factory Side - Migration Requirements
This document outlines what needs to be addressed in the **In-Factory** service to remove its coupling with Maestro.
> **Note:** This analysis is based on the Maestro codebase. For a complete analysis, the In-Factory repository should also be reviewed.
---
## Current Architecture (In-Factory → Maestro)
Based on the Maestro codebase analysis, **In-Factory** exposes the following gRPC services that Maestro consumes:
```mermaid
graph LR;
Maestro -->|gRPC| InFactory;
subgraph InFactory Services
CM[ConnectionManager]
ConM[ConnectorManager]
IN[Input]
NC[NetworkConfig]
CT[ConnectionTest]
TR[Transformation]
end
Maestro --> CM;
Maestro --> ConM;
Maestro --> IN;
Maestro --> NC;
Maestro --> CT;
Maestro --> TR;
```
---
## gRPC Services Exposed by In-Factory
### 1. ConnectionManager Service
**Package:** `ConnectionManager` from `@dadosfera/protospack-v2`
#### Write Services (`ConnectionManagerWriteServices`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `CreateConnection` | Create a new data connection | Maestro POST /connections |
| `UpdateConnection` | Update an existing connection | Maestro PUT /connections/:id |
| `DeleteConnection` | Delete a connection | Maestro DELETE /connections/:id |
#### Read Services (`ConnectionManagerReadServices`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `GetConnectionDetails` | Get details of a single connection | Maestro GET /connections/:id |
| `GetAllConnections` | List all connections for a customer | Maestro GET /connections |
| `GetConnectorAvailableConnectionsByCustomer` | Get available connections by connector | Maestro GET /connections/available/:connector_id |
| `ValidatePlatformConnections` | Validate connections against platform | Maestro internal |
---
### 2. ConnectorManager Service
**Package:** `ConnectorManager` from `@dadosfera/protospack-v2`
#### Write Services (`ConnectorManagerWriteServices`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `RegisterConnector` | Register a new connector | Maestro POST /connectors |
| `UploadFile` | Upload connector files | Maestro POST /connectors/uploads |
| `RegisterMultipleConnectorsWithoutImage` | Bulk register connectors | Maestro internal |
| `UpdateConnectorByID` | Update a connector | Maestro PUT /connectors/:id |
| `DeleteConnectorById` | Delete a connector | Maestro DELETE /connectors/:id |
#### Read Services (`ConnectorManagerReadServices`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `GetConnectors` | List all connectors | Maestro GET /connectors |
---
### 3. Input Service
**Package:** `Input` from `@dadosfera/protospack-v2`
#### Write Services (`InputWriteService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `CreateInput` | Create a new input configuration | Maestro POST /inputs |
| `UpdateInput` | Update input configuration | Maestro PATCH /inputs |
| `DeleteInput` | Delete an input | Maestro DELETE /inputs/:id |
#### Read Services (`InputReadService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `GetAvailableEntities` | Get available entities for a plugin | Maestro GET /inputs/available-entities/:plugin |
---
### 4. NetworkConfig Service
**Package:** `NetworkConfig` from `@dadosfera/protospack-v2`
#### Write Services (`NetworkConfigWriteService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `NetworkConfigCreate` | Create network configuration | Maestro POST /network-configs |
| `NetworkConfigUpdate` | Update network configuration | Maestro PUT /network-configs/:id |
| `NetworkConfigDelete` | Delete network configuration | Maestro DELETE /network-configs/:id |
#### Read Services (`NetworkConfigReadService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `NetworkConfigFindAll` | List all network configs | Maestro GET /network-configs |
| `NetworkConfigFindOneById` | Get network config by ID | Maestro GET /network-configs/:id |
---
### 5. ConnectionTest Service
**Package:** `ConnectionTest` from `@dadosfera/protospack-v2`
#### Read Services (`ConnectionTestReadService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `TestConnection` | Test a connection's connectivity | Maestro POST /connection-test |
---
### 6. Transformation Service
**Package:** `Transformation` from `@dadosfera/protospack-v2`
#### Write Services (`TransformationWriteService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| `CreateTransformation` | Create a transformation | Maestro POST /transformations |
| `UpdateTransformation` | Update a transformation | Maestro PUT /transformations/:id |
| `DeleteTransformation` | Delete a transformation | Maestro DELETE /transformations/:id |
#### Read Services (`TransformationReadService`)
| gRPC Method | Description | Called By |
|-------------|-------------|-----------|
| Various read operations | Get transformation details | Maestro GET /transformations |
---
## Proto Package Ownership
The proto definitions are managed in `@dadosfera/protospack-v2`:
```
@dadosfera/protospack-v2/
├── dist/lib/
│ ├── ConnectionManager/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ ├── ConnectorManager/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ ├── Input/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ ├── NetworkConfig/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ ├── ConnectionTest/
│ │ ├── interfaces/messages.ts
│ │ └── interfaces/entities.ts
│ └── Transformation/
│ ├── interfaces/messages.ts
│ └── interfaces/entities.ts
```
---
## In-Factory Migration Options
### Option 1: Expose REST API (Keep In-Factory, Add REST Layer)
**Changes Required in In-Factory:**
1. Add REST controllers for all services
2. Implement HTTP endpoints mirroring gRPC methods
3. Update deployment to expose HTTP port
4. Create OpenAPI documentation
**Pros:**
- Minimal architectural changes
- Can run both gRPC and REST in parallel during migration
- Maestro can switch to REST calls gradually
**Cons:**
- Still maintains service dependency
- Adds another communication layer
---
### Option 2: Move Logic to Maestro (Deprecate In-Factory for these features)
**Changes Required in In-Factory:**
1. Export database schema/migrations
2. Document all business logic
3. Provide data migration scripts
4. Deprecate gRPC endpoints after migration
**Changes Required in Maestro:**
1. Create database models
2. Implement repositories
3. Create service layer with same business logic
4. Run data migration
**Pros:**
- Removes runtime dependency completely
- Simplifies architecture
- One less service to maintain
**Cons:**
- Significant development effort
- Risk of business logic divergence during migration
- Database sharing concerns
---
### Option 3: Merge Services (Combine In-Factory into a larger service)
**Changes Required:**
1. Create new combined service
2. Migrate both In-Factory and relevant Maestro code
3. Update all clients
**Pros:**
- Clean architectural redesign
- Opportunity to optimize
**Cons:**
- Largest effort
- Risk of disruption
---
## Data Migration Considerations
### Entities Managed by In-Factory
Based on proto definitions, In-Factory manages:
1. **Connections**
- Connection credentials
- Connection metadata
- Customer associations
2. **Connectors**
- Connector definitions
- Connector images/files
- Plugin configurations
3. **Inputs**
- Input configurations
- Entity mappings
4. **Network Configs**
- Network configuration settings
- Security settings
5. **Transformations**
- Transformation definitions
- Transformation scripts
### Migration Steps
1. Export database schema from In-Factory
2. Create equivalent schema in target database
3. Write data migration scripts
4. Validate data integrity
5. Switch traffic
6. Decommission old service
---
## Recommended Investigation for In-Factory Team
1. **Check for Maestro dependencies in In-Factory**
- Does In-Factory call any Maestro APIs?
- Are there any shared databases?
- Any shared message queues?
2. **Document database schema**
- All tables related to connections, connectors, inputs, network-configs, transformations
- Foreign key relationships
- Indexes and constraints
3. **List all consumers**
- Besides Maestro, who else calls In-Factory?
- Are there other internal services?
- Any external integrations?
4. **Business logic documentation**
- Validation rules
- Business constraints
- Side effects (events, notifications, etc.)
---
## Timeline Considerations
| Phase | Description | Dependencies |
|-------|-------------|--------------|
| Phase 1 | Analysis & Planning | Both teams available |
| Phase 2 | Schema/API Design | Proto definitions finalized |
| Phase 3 | Implementation | Development resources |
| Phase 4 | Data Migration | Database access, downtime window |
| Phase 5 | Testing | QA resources, test environments |
| Phase 6 | Cutover | Deployment coordination |
| Phase 7 | Decommission | Monitoring, rollback plan |
---
## Questions for In-Factory Team
1. What database does In-Factory use? (PostgreSQL, MongoDB, etc.)
2. Are there any async operations? (message queues, event sourcing)
3. What is the current data volume for each entity type?
4. Are there any scheduled jobs or background processes?
5. What monitoring/alerting is in place?
6. Are there any data retention policies?
7. What is the backup/recovery strategy?
+486
View File
@@ -0,0 +1,486 @@
# Maestro ↔ PI-Factory Migration Map
This document maps all integration points between Maestro and PI-Factory that need to be addressed to remove the dependency.
## Architecture Overview
```mermaid
graph TD;
Frontend<-->Maestro;
Maestro<-->DUC;
Maestro<-->pi-factory;
Maestro<-->in-factory;
```
PI-Factory (`PIFACTORY_URL`) is responsible for:
- **Pipeline Management** - Create, read, update, delete pipelines
- **Catalog Services** - Data asset management, metadata, previews
- **Platform Interfaces** - Dataset cataloging operations
---
## Summary Table
| Category | Count | Impact Level |
|----------|-------|--------------|
| gRPC Client Configurations | 3 | HIGH |
| NestJS Modules | 3 | HIGH |
| REST Controllers/Endpoints | 3 (~50 endpoints) | HIGH |
| Service Classes | 4 | HIGH |
| Configuration Files | 4 | MEDIUM |
| Proto Package Dependencies | 2 | HIGH |
| Environment Variables | 1 | LOW |
---
## 1. gRPC CLIENT CONFIGURATIONS (Files to Migrate)
These files configure gRPC connections to PI-Factory services:
### 1.1 Catalog Client (`PIFACTORY_URL`)
**File:** `src/modules/catalog/catalog-client.ts`
```typescript
// Lines 11-12, 19
process.env.PIFACTORY_URL.startsWith('pi-factory:')
process.env.PIFACTORY_URL.includes('0.0.0.0')
url: process.env.PIFACTORY_URL
```
**Proto Services Used:**
- `Catalog.ProtoPackages.ReadPackage`
- `Catalog.ProtoPackages.WritePackage`
- `PlatformInterfaces.ProtoPackages.WritePackage`
### 1.2 Pipelines V2 Client (`PIFACTORY_URL`)
**File:** `src/modules/pipelinesV2/pipelines-client.ts`
```typescript
// Lines 13-14, 21
url: process.env.PIFACTORY_URL
```
**Proto Services Used:**
- `PipelineV2.ProtoPackages.ReadPackage`
- `PipelineV2.ProtoPackages.WritePackage`
### 1.3 Pipelines Client (Legacy) (`PIFACTORY_URL`)
**File:** `src/modules/pipelines/pipelines-client.ts`
```typescript
// Lines 10-11, 18
url: process.env.PIFACTORY_URL
```
**Proto Services Used:**
- `PipelinePackages` from `@dadosfera/protospack`
---
## 2. SERVICE CLASSES (Business Logic to Migrate)
### 2.1 Catalog Service
**File:** `src/modules/catalog/catalog.service.ts`
**gRPC Services Initialized:**
- `CatalogReadServices` (from `Catalog.ReadService`)
- `CatalogWriteServices` (from `Catalog.WriteService`)
- `PlatformInterfacesWriteServices` (from `PlatformInterfaces.WriteService`)
**gRPC Methods Called:**
| Method | Service | Description |
|--------|---------|-------------|
| `GetAllDataAssets()` | CatalogReadServices | Search/list data assets |
| `GetOneDataAsset()` | CatalogReadServices | Get single data asset by ID |
| `GetOneDataAssetByPipelineAndObject()` | CatalogReadServices | Get asset by pipeline/object |
| `GetDatasetDoc()` | CatalogReadServices | Get dataset documentation |
| `GetDatasetPreview()` | CatalogReadServices | Get data preview |
| `GetDatasetColumnsMetadata()` | CatalogReadServices | Get column metadata |
| `GetCustomerTags()` | CatalogReadServices | Get all tags for customer |
| `GetDatasetCatalogTask()` | CatalogReadServices | Get catalog task status |
| `GetRlsRules()` | CatalogReadServices | Get RLS rules |
| `GetOneRlsRule()` | CatalogReadServices | Get single RLS rule |
| `GetNimbusDashboards()` | CatalogReadServices | Get Nimbus dashboards |
| `CreateDataAsset()` | CatalogWriteServices | Create new data asset |
| `UpdateDataAsset()` | CatalogWriteServices | Update existing data asset |
| `DeleteDataAsset()` | CatalogWriteServices | Delete data asset |
| `ManagePermission()` | CatalogWriteServices | Manage asset permissions |
| `RevokePermission()` | CatalogWriteServices | Revoke asset permissions |
| `MakeAComment()` | CatalogWriteServices | Add comment to asset |
| `UpdateAComment()` | CatalogWriteServices | Update/delete comment |
| `TriggerDatasetCataloging()` | CatalogWriteServices | Trigger catalog process |
| `AddRlsRule()` | CatalogWriteServices | Add RLS rule |
| `RemoveRlsRule()` | CatalogWriteServices | Remove RLS rule |
| `RemoveRlsRulesByRlsId()` | CatalogWriteServices | Batch remove by RLS ID |
| `RemoveRlsRulesByDashboardId()` | CatalogWriteServices | Batch remove by dashboard |
| `GetPiiReporter()` | CatalogWriteServices | Get PII report data |
| `CatalogDataAssets()` | PlatformInterfacesWriteServices | Catalog datasets |
**Additional HTTP Calls to Nimbus:**
- `POST ${nimbusUrl}/api/catalog/data-docs/` - Create data docs
- `POST ${nimbusUrl}/api/catalog/table-metadata/` - Create table metadata
- `POST ${nimbusUrl}/api/catalog/column-metadata/` - Create column metadata
- `POST ${nimbusUrl}/api/catalog/data-preview/` - Create data preview
### 2.2 Pipelines V2 Service
**File:** `src/modules/pipelinesV2/pipelines.service.ts`
**gRPC Services Initialized:**
- `PipelineV2ReadService` (from `PipelineV2.ReadService`)
- `PipelineV2WriteService` (from `PipelineV2.WriteService`)
**gRPC Methods Called:**
| Method | Service | Description |
|--------|---------|-------------|
| `PipelineV2Create()` | PipelineV2WriteService | Create new pipeline |
| `PipelineV2Update()` | PipelineV2WriteService | Update pipeline |
| `PipelineV2Remove()` | PipelineV2WriteService | Delete pipeline |
| `PipelineV2UploadFile()` | PipelineV2WriteService | Initialize file upload |
| `PipelineV2CompleteUploadFile()` | PipelineV2WriteService | Complete file upload |
| `PipelineV2FindAll()` | PipelineV2ReadService | List all pipelines |
| `PipelineV2FindOne()` | PipelineV2ReadService | Get single pipeline |
| `PipelineV2FindObjects()` | PipelineV2ReadService | Get pipeline objects |
| `PipelineV2DownloadLogs()` | PipelineV2ReadService | Download pipeline logs |
| `PipelineV2GetDashboardUrl()` | PipelineV2ReadService | Get monitoring dashboard URL |
### 2.3 Pipelines Service (Legacy)
**File:** `src/modules/pipelines/pipelines.service.ts`
**Uses:** `PipelinesClientService`
**Methods:**
- `getPipelineStatus()` - Get pipeline execution status
- `runPipeline()` - Trigger pipeline execution
### 2.4 Pipelines Client Service (Legacy)
**File:** `src/modules/pipelines/client.service.ts`
**gRPC Methods Called:**
| Method | Service | Description |
|--------|---------|-------------|
| `getPipelineStatus()` | PipelineService | Get pipeline status |
| `triggerPipeline()` | PipelineService | Trigger pipeline run |
---
## 3. NESTJS MODULES (Module Registration)
| Module | File | Client Configuration | Exports |
|--------|------|---------------------|---------|
| CatalogModule | `src/modules/catalog/catalog.module.ts` | CatalogClientConfiguration | CatalogService |
| PipelinesV2Module | `src/modules/pipelinesV2/pipelines.module.ts` | PipelinesClientConfiguration | PipelinesService |
| PipelinesModule | `src/modules/pipelines/pipelines.module.ts` | PipelinesClientConfiguration | PipelinesService, PipelinesClientService |
**App Module Registration:** `src/app.module.ts` (Lines 20, 23, 25, 53-54, 59)
---
## 4. REST CONTROLLERS (API Endpoints to Migrate)
### 4.1 Catalog Controller
**File:** `src/modules/catalog/catalog.controller.ts`
**Base Path:** `/catalog`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| GET | `/catalog` | Search data assets |
| GET | `/catalog/download` | Download assets as CSV |
| GET | `/catalog/data-asset` | Get asset by pipeline/object |
| GET | `/catalog/data-asset/:id` | Get single data asset |
| GET | `/catalog/data-asset/rls/:id` | Get data asset RLS info |
| GET | `/catalog/data-asset/:id/columns-metadata` | Get column metadata |
| GET | `/catalog/data-asset/:id/preview` | Get data preview |
| GET | `/catalog/data-asset/:id/docs` | Get documentation |
| GET | `/catalog/tags` | Get all tags |
| PUT | `/catalog/data-asset/:id` | Update data asset |
| PUT | `/catalog/data-asset/:id/manage-permissions` | Manage permissions |
| PUT | `/catalog/data-asset/:id/revoke-permissions` | Revoke permissions |
| POST | `/catalog` | Create data asset |
| POST | `/catalog/data-asset/:id/docs` | Create documentation |
| POST | `/catalog/data-asset/:id/comment` | Add comment |
| POST | `/catalog/dataset-catalog-task` | Trigger catalog task |
| POST | `/catalog/rls-rule` | Add RLS rule |
| POST | `/catalog/register-dataset` | Register dataset with metadata |
| DELETE | `/catalog/data-asset/:id` | Delete data asset |
| DELETE | `/catalog/data-asset/:id/comment` | Delete comment |
| DELETE | `/catalog/rls-rule/:id` | Remove RLS rule |
| DELETE | `/catalog/rls-rule` | Batch remove RLS rules |
| GET | `/catalog/dataset-catalog-task/:session` | Get catalog task status |
| GET | `/catalog/rls-rule/:id` | Get single RLS rule |
| GET | `/catalog/rls-rule` | Get RLS rules |
| GET | `/catalog/nimbus-dashboards` | Get Nimbus dashboards |
| GET | `/catalog/pii-reporter` | Get PII report |
**Total: 27 endpoints**
### 4.2 Pipelines V2 Controller
**File:** `src/modules/pipelinesV2/pipelines.controller.ts`
**Base Path:** `/pipelinesV2`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| GET | `/pipelinesV2/monitoring-dashboard` | Get monitoring dashboard URL |
| GET | `/pipelinesV2` | List all pipelines |
| GET | `/pipelinesV2/download-logs` | Download pipeline logs |
| GET | `/pipelinesV2/:id` | Get single pipeline |
| GET | `/pipelinesV2/:id/config` | Get pipeline properties |
| GET | `/pipelinesV2/:id/objects` | Get pipeline objects |
| GET | `/pipelinesV2/:id/status` | Get pipeline status (legacy) |
| POST | `/pipelinesV2` | Create pipeline |
| POST | `/pipelinesV2/init-upload` | Initialize file upload |
| POST | `/pipelinesV2/complete-upload` | Complete file upload |
| POST | `/pipelinesV2/file` | Upload file pipeline |
| POST | `/pipelinesV2/start/:id` | Start pipeline |
| PATCH | `/pipelinesV2/:id` | Update pipeline |
| PUT | `/pipelinesV2/:id` | Update pipeline (deprecated) |
| DELETE | `/pipelinesV2/:id` | Delete pipeline |
**Total: 15 endpoints**
### 4.3 Pipelines Controller (Legacy)
**File:** `src/modules/pipelines/pipelines.controller.ts`
**Base Path:** `/pipelines`
| HTTP Method | Endpoint | Description |
|-------------|----------|-------------|
| POST | `/pipelines/start/:id` | Start pipeline (deprecated) |
| GET | `/pipelines/:id/status` | Get pipeline status (deprecated) |
**Total: 2 endpoints (deprecated)**
---
## 5. CONFIGURATION FILES
### 5.1 Helm Chart Values (Production)
**File:** `deploy/helm-chart/values.yaml`
```yaml
# Line 40
maestro:
pi_factory_url: pi-factory.dadosfera.ai
```
### 5.2 Helm Chart Values (Staging)
**File:** `deploy/helm-chart/values-stg.yaml`
```yaml
# Line 4
maestro:
pi_factory_url: pi-factory.stg.dadosfera.ai
```
### 5.3 Helmfiles (Production)
**File:** `deploy/helmfiles/prd.yaml`
```yaml
# Lines 14, 38
maestro.pi_factory_url: pi-factory.dadosfera.ai
```
### 5.4 Deployment Template
**File:** `deploy/helm-chart/templates/deployment.yaml`
```yaml
# Line 85
- name: PIFACTORY_URL
value: {{ .Values.maestro.pi_factory_url }}
```
### 5.5 Environment Type Definition
**File:** `environment.d.ts`
```typescript
// Line 10
PIFACTORY_URL: string;
```
---
## 6. PROTO PACKAGE DEPENDENCIES
### 6.1 Package.json
**File:** `package.json`
```json
{
"@dadosfera/protospack": "2.5.3", // Legacy pipelines
"@dadosfera/protospack-v2": "3.38.0-beta.14" // PipelineV2, Catalog, PlatformInterfaces
}
```
### 6.2 Proto Imports from `@dadosfera/protospack-v2` (PI-Factory related)
| Import Path | Used In |
|-------------|---------|
| `Catalog` | catalog/catalog-client.ts, catalog/catalog.service.ts |
| `Catalog/interfaces/messages` | catalog/catalog.service.ts, catalog/catalog.controller.ts, catalog/dtos |
| `PlatformInterfaces` | catalog/catalog-client.ts, catalog/catalog.service.ts |
| `PipelineV2` | pipelinesV2/pipelines-client.ts, pipelinesV2/pipelines.service.ts |
| `PipelineV2/interfaces/messages` | pipelinesV2/pipelines.service.ts, pipelinesV2/pipelines.controller.ts |
### 6.3 Proto Imports from `@dadosfera/protospack` (Legacy)
| Import Path | Used In |
|-------------|---------|
| `PipelinePackages` | pipelines/pipelines-client.ts |
| `PipelineProtoFilePath` | pipelines/pipelines-client.ts |
| `PipelineServicesNames` | pipelines/client.service.ts |
| `PipelinesServiceInterface` | pipelines/client.service.ts |
---
## 7. FILES TO MODIFY/DELETE (Summary)
### High Priority - Core Integration Files
```
src/modules/catalog/catalog-client.ts → DELETE or REPLACE
src/modules/catalog/catalog.service.ts → REPLACE with local implementation
src/modules/pipelinesV2/pipelines-client.ts → DELETE or REPLACE
src/modules/pipelinesV2/pipelines.service.ts → REPLACE with local implementation
src/modules/pipelines/pipelines-client.ts → DELETE or REPLACE
src/modules/pipelines/client.service.ts → REPLACE with local implementation
src/modules/pipelines/pipelines.service.ts → REPLACE with local implementation
```
### Medium Priority - Module Registration
```
src/modules/catalog/catalog.module.ts → UPDATE imports
src/modules/pipelinesV2/pipelines.module.ts → UPDATE imports
src/modules/pipelines/pipelines.module.ts → UPDATE imports
src/app.module.ts → UPDATE if module structure changes
```
### Low Priority - Configuration
```
deploy/helm-chart/values.yaml → REMOVE pi_factory_url
deploy/helm-chart/values-stg.yaml → REMOVE pi_factory_url
deploy/helmfiles/prd.yaml → REMOVE pi_factory_url references
deploy/helm-chart/templates/deployment.yaml → REMOVE PIFACTORY_URL env var
environment.d.ts → REMOVE PIFACTORY_URL type
README.md → UPDATE architecture diagram
```
---
## 8. DEPENDENCY COUNT BY MODULE
| Module | Files | gRPC Calls | REST Endpoints |
|--------|-------|------------|----------------|
| Catalog | 3 | 24+ | 27 |
| PipelinesV2 | 3 | 10 | 15 |
| Pipelines (Legacy) | 3 | 2 | 2 |
| **TOTAL** | **9** | **36+** | **44** |
---
## 9. COMPARISON: PI-Factory vs In-Factory
| Aspect | PI-Factory | In-Factory |
|--------|------------|------------|
| Environment Variable | `PIFACTORY_URL` | `INFACTORY_URL` |
| Modules | 3 | 6 |
| gRPC Calls | 36+ | 27+ |
| REST Endpoints | 44 | 25 |
| Complexity | HIGH | MEDIUM-HIGH |
| Domain | Pipelines, Catalog | Connections, Connectors, Inputs |
---
## 10. DATA MODELS (Proto Messages Used)
### Catalog Messages
- `CreateDataAssetRequest` / `CreateDataAssetResponse`
- `GetAllDataAssetsRequest` / `GetAllDataAssetsResponse`
- `GetOneDataAssetRequest` / Response
- `UpdateDataAssetRequest` / Response
- `DeleteDataAssetRequest` / Response
- `ManagePermissionRequest` / Response
- `RevokePermissionRequest` / Response
- `MakeACommentRequest` / Response
- `UpdateACommentRequest` / Response
- `TriggerDatasetCatalogingRequest` / Response
- `GetDatasetCatalogTaskRequest` / Response
- `AddRlsRuleRequest` / Response
- `RemoveRlsRuleRequest` / Response
- `GetRlsRulesRequest` / Response
- `GetNimbusDashboardsRequest` / Response
- `PiiMetadata`
- `RegisterDatasetWithMetatadaRequest`
### PipelineV2 Messages
- `PipelineV2CreateRequest` / `PipelineV2CreateResponse`
- `PipelineV2FindAllRequest` / `PipelineV2FindAllResponse`
- `PipelineV2FindOneRequest` / `PipelineV2FindOneResponse`
- `PipelineV2UpdateRequest` / `PipelineV2UpdateResponse`
- `PipelineV2RemoveRequest` / Response
- `PipelineV2UploadFileRequest` / Response
- `PipelineV2CompleteUploadFileRequest` / Response
- `PipelineV2FindObjectsRequest` / Response
- `PipelineV2DownloadLogsRequest` / Response
- `PipelineV2GetDashboardUrlRequest` / Response
### Platform Interfaces Messages
- `CatalogDataAssetsRequest` / Response
---
## 11. MIGRATION STRATEGY OPTIONS
### Option A: Move PI-Factory functionality INTO Maestro
**Pros:**
- Single service to maintain
- No network latency for these operations
- Simpler deployment
**Cons:**
- Significantly increases Maestro's responsibility
- Requires database access from Maestro
- Large refactoring effort (44 endpoints)
### Option B: Create REST API wrapper in PI-Factory
**Pros:**
- Minimal changes to Maestro
- Can migrate incrementally
**Cons:**
- Still maintains dependency
- Additional REST→gRPC translation layer
### Option C: Direct database access from Maestro
**Pros:**
- Removes runtime dependency
- Better performance
**Cons:**
- Shared database coupling
- Complex migration
---
## 12. RECOMMENDED MIGRATION ORDER
Given the complexity, we recommend migrating in this order:
1. **Legacy Pipelines** (2 endpoints, deprecated) - Lowest risk
2. **PipelinesV2** (15 endpoints) - Core pipeline functionality
3. **Catalog** (27 endpoints) - Most complex, migrate last
### Phase 1: Legacy Pipelines (Deprecated)
- Remove `/pipelines/start/:id`
- Remove `/pipelines/:id/status`
- Update all clients to use `/pipelinesV2/*` endpoints
### Phase 2: PipelinesV2
- Migrate pipeline CRUD operations
- Migrate file upload functionality
- Migrate monitoring dashboard
### Phase 3: Catalog
- Migrate data asset CRUD
- Migrate permissions management
- Migrate RLS rules
- Migrate dataset registration
- Migrate PII reporting
---
## NEXT STEPS
1. **Decide on migration strategy** (Option A, B, or C)
2. **Deprecate legacy pipelines module** first
3. **Create database schema** if moving to Option A
4. **Implement local services** one module at a time
5. **Update tests** for each migrated module
6. **Update deployment configs** to remove PIFACTORY_URL
7. **Coordinate with PI-Factory team** for data migration
@@ -74,6 +74,8 @@ spec:
value: "logstash-pipelines.dadosfera.ai"
- name: LOGGER_GELF_PORT
value: "{{ .Values.maestro.logger_gelf_port }}"
- name: LOGGER_CONSOLE_EXTRA
value: "true"
- name: NIMBUS_BASE_URL
value: "http://nimbus-api"
- name: NPM_TOKEN
@@ -4,9 +4,18 @@ metadata:
annotations:
nginx.ingress.kubernetes.io/whitelist-source-range: "69.49.241.121/32" # hostgator ip
nginx.ingress.kubernetes.io/proxy-body-size: "0"
nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
nginx.ingress.kubernetes.io/proxy-connect-timeout: "300"
nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
nginx.ingress.kubernetes.io/server-snippet: |
underscores_in_headers on;
ignore_invalid_headers on;
nginx.ingress.kubernetes.io/proxy-buffer-size: "16k"
nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "64k"
{{- if .Values.maestro.restricted_ip}}
nginx.ingress.kubernetes.io/whitelist-source-range: {{ .Values.maestro.restricted_ip }}
{{- end }}
generation: 1
labels:
+2 -9
View File
@@ -7,17 +7,10 @@ maestro:
open_customer_id: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
open_group_id: e3f98a2f-7748-4981-8505-7695c8ca8218
cookie_secret: "ff7bc13823edb2ae50d248e5780bddc9d4b31c36"
redis_database: "1"
hostname: maestro.stg.dadosfera.ai
replicaCount: 1
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: name
operator: In
values:
- general
affinity: null
+3 -3
View File
@@ -45,7 +45,7 @@ maestro:
open_group_id: 401573bb-334f-44b2-b30e-88d4cea31ae9
dedicated_proxy: ""
restricted_ip: ""
redis_host: "product-redis-prd.z4xvqj.0001.use1.cache.amazonaws.com"
redis_host: "aaapzppmlyamkocqwstpo7zvopczyyiyuy6xzm2g6c5k4mq3a66be4a-0.redis.sa-saopaulo-1.oci.oraclecloud.com"
redis_port: "6379"
redis_database: "0"
cookie_secret: "13cc5e136d3074bcc05bec8697092ec1f5f376bf"
@@ -61,7 +61,7 @@ affinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: applications
- key: name
operator: In
values:
- general
- product
+5 -2
View File
@@ -1,4 +1,4 @@
charts:
releases:
- name: maestro
chart: ../helm-chart
values:
@@ -49,5 +49,8 @@ charts:
value: dea2c27f-0973-4588-a2e0-9e31b64c7ffd
- name: replicaCount
value: 1
# 10.70.0.0/16 internal network
# 137.131.167.254/32 loadbalancer
# 159.112.184.81/32 cluster ip for the uptime request ingest
- name: maestro.restricted_ip
value: "177.52.172.0/24,189.84.160.157/32,186.237.171.146/32,57.151.113.140/30"
value: "177.52.172.0/24, 189.84.160.157/32, 186.237.171.146/32, 137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
+21 -34
View File
@@ -7,37 +7,24 @@ charts:
# Environment to test Network Policies
# - name: private-maestro
# chart: ../helm-chart
# values:
# - ../helm-chart/values.yaml
# set:
# - name: app_name
# value: maestro-private
# - name: maestro.env
# value: stg
# - name: maestro.duc_url
# value: duc.stg.dadosfera.ai
# - name: hostname
# value: private-maestro.stg.dadosfera.ai
# - name: maestro.pi_factory_url
# value: pi-factory.dadosfera.ai
# - name: maestro.in_factory_url
# value: in-factory.stg.dadosfera.ai
# - name: maestro.tr_factory_url
# value: in-factory.dadosfera.ai
# - name: maestro.open_customer_id
# value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d
# - name: maestro.open_group_id
# value: e3f98a2f-7748-4981-8505-7695c8ca8218
# # Customer id
# - name: maestro.dedicated_proxy
# value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
# - name: replicaCount
# value: 1
# - name: affinity
# value: null
# - name: resources
# value: null
# - name: maestro.restricted_ip
# value: "57.151.113.140/30"
- name: private-maestro
chart: ../helm-chart
values:
- ../helm-chart/values.yaml
- ../helm-chart/values-stg.yaml
set:
- name: app_name
value: maestro-private
- name: hostname
value: private-maestro.stg.dadosfera.ai
# Customer id
- name: maestro.dedicated_proxy
value: 14d52fd4-d83d-4cdd-be34-bf11cc28b3bd
- name: replicaCount
value: 1
- name: affinity
value: null
- name: resources
value: null
- name: maestro.restricted_ip
value: "137.131.167.254/32, 10.70.0.0/16, 159.112.184.81/32, 10.244.0.0/16"
+60 -9
View File
@@ -42,6 +42,33 @@
]
}
},
"/auth/sign-out": {
"post": {
"operationId": "AuthController_signOut",
"parameters": [
{
"name": "dadosfera-lang",
"in": "header",
"required": false,
"schema": {
"enum": [
"pt-br",
"en-us"
],
"type": "string"
}
}
],
"responses": {
"204": {
"description": ""
}
},
"tags": [
"Auth"
]
}
},
"/auth/refresh-access-token": {
"post": {
"operationId": "AuthController_refreshAccessToken",
@@ -3415,9 +3442,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3467,9 +3491,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -3519,9 +3540,6 @@
"PipelinesV2"
],
"security": [
{
"access-token": []
},
{
"access-token": []
}
@@ -6251,6 +6269,39 @@
]
}
},
"/customers/{id}/theme/reset": {
"post": {
"operationId": "ThemeController_resetTheme",
"parameters": [
{
"name": "id",
"required": true,
"in": "path",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/CustomerThemeResponse"
}
}
}
},
"201": {
"description": ""
}
},
"tags": [
"Theme"
]
}
},
"/network-policy": {
"get": {
"operationId": "NetworkPolicyController_getNetworks",
+42 -35
View File
@@ -12,7 +12,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@dadosfera/protospack-v2": "3.38.0-beta.14",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -57,8 +57,8 @@
"swagger-ui-express": "^4.6.3"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/cookie-parser": "^1.4.9",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
@@ -850,6 +850,7 @@
"integrity": "sha512-vMqyb7XCDMPvJFFOaT9kxtiRh42GwlZEg1/uIgtZshS5a/8OaduUfCi7kynKgc3Tw/6Uo2D+db9qBttghhmxwQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@ampproject/remapping": "^2.2.0",
"@babel/code-frame": "^7.26.2",
@@ -1408,9 +1409,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.38.0-beta.10",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.10.tgz",
"integrity": "sha512-4miwovVFHtBY1VTHdW8BVmSB8m+LXfIA8uigHbeo8IwreMtTHuLpvfenT4MBSPkdVmlo9+0XBKKf+PaSqtdMAg==",
"version": "3.38.0-beta.14",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.38.0-beta.14.tgz",
"integrity": "sha512-BiE1fUIHuam3cJjAGIknsRNIlf1Z0JHJOV/VZ2pyAp5mIkGOSoNFBAMXJZb+VMTnd2xsp+vPxNnWVu7OJ/CvMQ==",
"license": "ISC",
"dependencies": {
"@grpc/grpc-js": "^1.9.3",
@@ -2260,6 +2261,7 @@
"resolved": "https://registry.npmjs.org/@nestjs/common/-/common-9.4.3.tgz",
"integrity": "sha512-Gd6D4IaYj01o14Bwv81ukidn4w3bPHCblMUq+SmUmWLyosK+XQmInCS09SbDDZyL8jy86PngtBLTdhJ2bXSUig==",
"license": "MIT",
"peer": true,
"dependencies": {
"iterare": "1.2.1",
"tslib": "2.5.3",
@@ -2338,6 +2340,7 @@
"integrity": "sha512-Qi63+wi55Jh4sDyaj5Hhx2jOpKqT386aeo+VOKsxnd+Ql9VvkO/FjmuwBGUyzkJt29ENYc+P0Sx/k5LtstNpPQ==",
"hasInstallScript": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@nuxtjs/opencollective": "0.3.2",
"fast-safe-stringify": "2.1.1",
@@ -2475,6 +2478,7 @@
"resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-9.4.3.tgz",
"integrity": "sha512-FpdczWoRSC0zz2dNL9u2AQLXKXRVtq4HgHklAhbL59X0uy+mcxhlSThG7DHzDMkoSnuuHY8ojDVf7mDxk+GtCw==",
"license": "MIT",
"peer": true,
"dependencies": {
"body-parser": "1.20.2",
"cors": "2.8.5",
@@ -2838,6 +2842,7 @@
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
"integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
"license": "Apache-2.0",
"peer": true,
"engines": {
"node": ">=8.0.0"
}
@@ -2997,6 +3002,7 @@
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.6.1.tgz",
"integrity": "sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==",
"license": "MIT",
"peer": true,
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
@@ -3793,6 +3799,7 @@
"integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@types/body-parser": "*",
"@types/express-serve-static-core": "^4.17.33",
@@ -3873,6 +3880,7 @@
"integrity": "sha512-4dRxkS/AFX0c5XW6IPMNOydLn2tEhNhJV7DnYK+0bjoJZ+QTmfucBlihX7aoEsh/ocYtkLC73UbnBXBXIxsULA==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"jest-diff": "^27.0.0",
"pretty-format": "^27.0.0"
@@ -3936,7 +3944,8 @@
"version": "16.18.126",
"resolved": "https://registry.npmjs.org/@types/node/-/node-16.18.126.tgz",
"integrity": "sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/@types/oauth": {
"version": "0.9.6",
@@ -4166,6 +4175,7 @@
"integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==",
"dev": true,
"license": "BSD-2-Clause",
"peer": true,
"dependencies": {
"@typescript-eslint/scope-manager": "5.62.0",
"@typescript-eslint/types": "5.62.0",
@@ -4512,6 +4522,7 @@
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.1.tgz",
"integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==",
"license": "MIT",
"peer": true,
"bin": {
"acorn": "bin/acorn"
},
@@ -4608,6 +4619,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
"integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"json-schema-traverse": "^1.0.0",
@@ -5236,6 +5248,7 @@
}
],
"license": "MIT",
"peer": true,
"dependencies": {
"caniuse-lite": "^1.0.30001688",
"electron-to-chromium": "^1.5.73",
@@ -5342,6 +5355,7 @@
"resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-5.7.6.tgz",
"integrity": "sha512-wBxnBHjDxF1RXpHCBD6HGvKER003Ts7IIm0CHpggliHzN1RZditb7rXoduE1rplc2DEFYKxhLKgFuchXMJje9w==",
"license": "MIT",
"peer": true,
"dependencies": {
"eventemitter3": "^5.0.1",
"lodash.clonedeep": "^4.5.0",
@@ -5565,13 +5579,15 @@
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz",
"integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==",
"license": "MIT"
"license": "MIT",
"peer": true
},
"node_modules/class-validator": {
"version": "0.14.1",
"resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.1.tgz",
"integrity": "sha512-2VEG9JICxIqTpoK1eMzZqaV+u/EiwEJkMGzTrZf6sU/fwsnOITVgYJ8yojSy6CaXtO9V0Cc6ZQZ8h8m4UBuLwQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"@types/validator": "^13.11.8",
"libphonenumber-js": "^1.10.53",
@@ -6235,7 +6251,8 @@
"version": "0.0.1425554",
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz",
"integrity": "sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==",
"license": "BSD-3-Clause"
"license": "BSD-3-Clause",
"peer": true
},
"node_modules/dezalgo": {
"version": "1.0.4",
@@ -6512,7 +6529,6 @@
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz",
"integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.8"
}
@@ -6700,6 +6716,7 @@
"deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@eslint-community/eslint-utils": "^4.2.0",
"@eslint-community/regexpp": "^4.6.1",
@@ -7046,7 +7063,6 @@
"resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
"integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
"license": "MIT",
"peer": true,
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
@@ -7093,7 +7109,6 @@
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
"integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
"license": "MIT",
"peer": true,
"dependencies": {
"bytes": "3.1.2",
"content-type": "~1.0.5",
@@ -7118,7 +7133,6 @@
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz",
"integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.6"
}
@@ -7128,7 +7142,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -7137,22 +7150,19 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/express/node_modules/path-to-regexp": {
"version": "0.1.12",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz",
"integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/express/node_modules/qs": {
"version": "6.13.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
"integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"license": "BSD-3-Clause",
"peer": true,
"dependencies": {
"side-channel": "^1.0.6"
},
@@ -7181,8 +7191,7 @@
"url": "https://feross.org/support"
}
],
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/external-editor": {
"version": "3.1.0",
@@ -7417,7 +7426,6 @@
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz",
"integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==",
"license": "MIT",
"peer": true,
"dependencies": {
"debug": "2.6.9",
"encodeurl": "~2.0.0",
@@ -7436,7 +7444,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -7445,8 +7452,7 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/find-up": {
"version": "5.0.0",
@@ -8240,6 +8246,7 @@
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.6.1.tgz",
"integrity": "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==",
"license": "MIT",
"peer": true,
"dependencies": {
"@ioredis/commands": "^1.1.1",
"cluster-key-slot": "^1.1.0",
@@ -8581,6 +8588,7 @@
"integrity": "sha512-Yn0mADZB89zTtjkPJEXwrac3LHudkQMR+Paqa8uxJHCBr9agxztUifWCyiYrjhMPBoUVBjyny0I7XH6ozDr7QQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@jest/core": "^27.5.1",
"import-local": "^3.0.2",
@@ -9870,7 +9878,6 @@
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz",
"integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==",
"license": "MIT",
"peer": true,
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
@@ -10628,6 +10635,7 @@
"resolved": "https://registry.npmjs.org/passport/-/passport-0.6.0.tgz",
"integrity": "sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug==",
"license": "MIT",
"peer": true,
"dependencies": {
"passport-strategy": "1.x.x",
"pause": "0.0.1",
@@ -11017,6 +11025,7 @@
"integrity": "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==",
"dev": true,
"license": "MIT",
"peer": true,
"bin": {
"prettier": "bin-prettier.js"
},
@@ -11519,7 +11528,8 @@
"version": "0.1.14",
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.14.tgz",
"integrity": "sha512-ZhYeb6nRaXCfhnndflDK8qI6ZQ/YcWZCISRAWICW9XYqMUwjZM9Z0DveWX/ABN01oxSHwVxKQmxeYZSsm0jh5A==",
"license": "Apache-2.0"
"license": "Apache-2.0",
"peer": true
},
"node_modules/relative-microtime": {
"version": "2.0.0",
@@ -11706,6 +11716,7 @@
"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz",
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
"license": "Apache-2.0",
"peer": true,
"dependencies": {
"tslib": "^2.1.0"
}
@@ -11767,6 +11778,7 @@
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"license": "MIT",
"peer": true,
"dependencies": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
@@ -11810,7 +11822,6 @@
"resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz",
"integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==",
"license": "MIT",
"peer": true,
"dependencies": {
"debug": "2.6.9",
"depd": "2.0.0",
@@ -11835,7 +11846,6 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"license": "MIT",
"peer": true,
"dependencies": {
"ms": "2.0.0"
}
@@ -11844,15 +11854,13 @@
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/send/node_modules/encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==",
"license": "MIT",
"peer": true,
"engines": {
"node": ">= 0.8"
}
@@ -11861,8 +11869,7 @@
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT",
"peer": true
"license": "MIT"
},
"node_modules/serialize-javascript": {
"version": "6.0.2",
@@ -11878,7 +11885,6 @@
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz",
"integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==",
"license": "MIT",
"peer": true,
"dependencies": {
"encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
@@ -12856,6 +12862,7 @@
"integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
"dev": true,
"license": "MIT",
"peer": true,
"dependencies": {
"@cspotcode/source-map-support": "^0.8.0",
"@tsconfig/node10": "^1.0.7",
@@ -13085,6 +13092,7 @@
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.5.tgz",
"integrity": "sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==",
"license": "Apache-2.0",
"peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -13412,7 +13420,6 @@
"resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-4.3.0.tgz",
"integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==",
"license": "MIT",
"peer": true,
"dependencies": {
"@types/json-schema": "^7.0.9",
"ajv": "^8.9.0",
+2 -2
View File
@@ -30,7 +30,7 @@
"@aws-sdk/client-secrets-manager": "^3.414.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack": "2.5.3",
"@dadosfera/protospack-v2": "3.38.0-beta.10",
"@dadosfera/protospack-v2": "3.38.0-beta.14",
"@grpc/grpc-js": "^1.9.3",
"@grpc/proto-loader": "^0.7.9",
"@nestjs/cli": "^9.5.0",
@@ -78,8 +78,8 @@
"multer": "1.4.5-lts.1"
},
"devDependencies": {
"@types/cookie-parser": "^1.4.9",
"@types/cache-manager": "^4.0.6",
"@types/cookie-parser": "^1.4.9",
"@types/express": "^4.17.17",
"@types/express-session": "^1.18.1",
"@types/jest": "27.0.2",
+20
View File
@@ -0,0 +1,20 @@
import jwt, { JwtPayload } from 'jsonwebtoken';
export function extractUserFrom(aRawJwt: string) {
const decodedToken = jwt.decode(aRawJwt, {
complete: true,
});
const payload = decodedToken.payload as JwtPayload;
return {
user_id: payload.user_id,
username: payload.username,
permissions: payload.permissions,
customer_id: payload.customer_id,
customer_name: payload.customer_name,
customer_tier: payload.customer_tier,
customer_modules: payload.customer_modules,
access_token: aRawJwt,
}
}
+57
View File
@@ -116,6 +116,44 @@ export const PERMISSIONS_GROUPS = {
},
},
},
IMPORT_FILES: {
title: {
'pt-br': 'Coletar | Importar arquivos',
'en-us': 'Collect | Import files',
'es-es': 'Colecta | Importar archivos',
},
permissions: {
VIEW: {
seqid: 48,
claim: 'import-file:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Importar arquivos',
'en-us': 'Import files',
'es-es': 'Importar archivos',
},
},
},
},
AI_CHAT: {
title: {
'pt-br': 'AutodriveDDF',
'en-us': 'AutodriveDDF',
'es-es': 'AutodriveDDF',
},
permissions: {
VIEW: {
seqid: 49,
claim: 'ai-chat:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'AutodriveDDF',
'en-us': 'AutodriveDDF',
'es-es': 'AutodriveDDF',
},
},
},
},
CONNECTION: {
title: {
'pt-br': 'Coletar | Fontes de dados',
@@ -352,6 +390,25 @@ export const PERMISSIONS_GROUPS = {
},
},
},
LINEAGE: {
title: {
'pt-br': 'Explorar | Linhagem',
'en-us': 'Explore | Lineage',
'es-es': 'Explorar | Linaje',
},
permissions: {
VIEW: {
seqid: 50,
claim: 'lineage:view',
usage: PermissionUsages.PUBLIC,
name: {
'pt-br': 'Acessar ao módulo de Linhagem',
'en-us': 'Access to Lineage module',
'es-es': 'Acceda al módulo de Linaje',
},
}
},
},
EMBED: {
title: {
'pt-br': 'Analisar | Incorporação',
+24 -4
View File
@@ -18,21 +18,41 @@ async function bootstrap() {
});
const logger = new DadosferaLogger();
const corsOrigins = [];
if (process.env.ENV === 'local') {
corsOrigins.push('http://localhost:4200');
} else {
corsOrigins.push(
'https://app.stg.dadosfera.ai',
'https://app.dadosfera.ai',
'https://private-frontend.stg.dadosfera.ai',
'https://unimed.dadosfera.ai',
'https://boston-scientific.dadosfera.ai',
'https://plataforma.dadosfera.ai'
);
}
const app = await NestFactory.create(AppModule, {
logger,
cors: {
origin: '*',
origin: corsOrigins,
methods: 'GET,HEAD,PUT,PATCH,POST,DELETE',
preflightContinue: false,
optionsSuccessStatus: 204,
credentials: true
credentials: true,
},
});
app.use(helmet());
app.use(cookieParser(process.env.COOKIE_SECRET));
if (process.env.ENV === 'prd') {
if (process.env.ENV !== 'local') {
app.use('/catalog/register-dataset', json({ limit: '10mb' }));
app.use('/catalog/register-dataset', urlencoded({ extended: true, limit: '10mb' }));
app.use(
'/catalog/register-dataset',
urlencoded({ extended: true, limit: '10mb' }),
);
}
configureSwagger(app);
+6 -2
View File
@@ -1,4 +1,4 @@
import { Controller, Post, Body, Put, Get} from '@nestjs/common';
import { Controller, Body, Put, Get, NotFoundException} from '@nestjs/common';
import { AssignService } from './assign.service';
import { CreateAssignDto } from './dto/create-assign.dto';
import { Authenticated, RequireModule, RequireSomePermission } from 'src/decorators/authentication.decorator';
@@ -28,6 +28,10 @@ export class AssignController {
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED_ASSIGNED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
try {
return await this.assignService.get(metadata);
} catch (error) {
throw new NotFoundException(error.message)
}
}
}
+8 -37
View File
@@ -54,7 +54,6 @@ import jwt, { JwtPayload } from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
import { Cookie } from 'express-session';
type CookiesValues = {
accessToken?: string;
@@ -114,6 +113,7 @@ export class AuthController {
return res.send(data);
} catch (error) {
this.logger.error('/auth - SignIn - ERROR', error);
throw error;
}
}
@@ -497,29 +497,10 @@ export class AuthController {
const userId = req.cookies['ddf-user-id'];
this.logger.info('Has cookie: ' + Boolean(accessToken))
let payload: any;
let userInfo: any = {};
try {
// Decodifica e valida o JWT de acesso
const decoded: any = accessToken && jwt.decode(accessToken, { complete: true });
if (!decoded) throw new Error('Invalid token')
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue) throw new Error('Public key not found');
jwt.verify(accessToken, pemValue);
payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
const userDto = await this.authClient.extractUserFrom(accessToken);
return res.status(200).json(userDto);
} catch (err) {
this.logger.error(err.message);
const refreshToken = req.cookies['ddf-refresh-auth'];
@@ -544,18 +525,8 @@ export class AuthController {
userId
});
// Decodifica novo token
const decoded: any = jwt.decode(data.accessToken, { complete: true });
const payload = decoded.payload;
userInfo = {
id: payload.user_id,
name: payload.username,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return res.status(200).json(userInfo);
const userDto = await this.authClient.extractUserFrom(accessToken);
return res.status(200).json(userDto);
} catch (refreshErr) {
this.logger.error(refreshErr)
return res.status(401).json({ error: 'Not authenticated' });
@@ -572,7 +543,7 @@ export class AuthController {
this.logger.info('Set Cookie ddf-auth')
res.cookie('ddf-auth', data.accessToken, {
domain: 'stg.dadosfera.ai',
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
@@ -583,7 +554,7 @@ export class AuthController {
if (data.refreshToken) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-refresh-auth', data.refreshToken, {
domain: 'stg.dadosfera.ai',
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
@@ -594,7 +565,7 @@ export class AuthController {
if (data.userId) {
this.logger.info('Set Cookie ddf-refresh-auth')
res.cookie('ddf-user-id', data.userId, {
domain: 'stg.dadosfera.ai',
domain: '.dadosfera.ai',
maxAge: exp,
httpOnly: true,
secure: true,
+55 -8
View File
@@ -4,7 +4,7 @@ import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { lastValueFrom } from 'rxjs';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import { AuthProtoService as AuthServiceInterface, IdentityProviderProtoService, UsersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import {
AuthSnowflakeSignInRequest,
AuthSignInRequest,
@@ -21,18 +21,18 @@ import {
} from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { DucClient } from '../duc/client.config';
import { Metadata } from '@grpc/grpc-js';
import { BulkEditResponse } from './dtos/login';
import { BulkEditResponse, UserDTO } from './dtos/login';
import jwt from 'jsonwebtoken';
import { PackTheMetadata } from 'src/utils/PackTheMetadata';
@Injectable()
export class AuthClientService implements OnModuleInit {
logger: DadosferaLogger;
private authService: AuthServiceInterface;
private identityProviderService: IdentityProviderProtoService;
private userService: UsersProtoService;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
@@ -46,8 +46,8 @@ export class AuthClientService implements OnModuleInit {
ProtoServices.AuthProtoService,
);
this.identityProviderService = this.grpcClient.getService<IdentityProviderProtoService>(
ProtoServices.IdentityProviderProtoService,
this.userService = this.grpcClient.getService<UsersProtoService>(
ProtoServices.UsersProtoService,
);
}
@@ -283,4 +283,51 @@ export class AuthClientService implements OnModuleInit {
throw error;
}
}
private async getUser(id: string, metadata: Metadata) {
this.logger.info('getUser');
return await lastValueFrom(
this.userService.UserFindOneById({ id }, metadata),
);
}
public async extractUserFrom(token: string) {
const decoded: any = token && jwt.decode(token, { complete: true });
if (!decoded) throw new Error('Invalid token');
const { kid } = decoded.header;
// Busca a chave pública
const { keys } = await this.getPublicKeys();
const pemValue = keys.find((k) => k.kid === kid)?.pem;
if (!pemValue)
throw new Error('Public key not found');
jwt.verify(token, pemValue);
const payload = decoded.payload;
const metadata = PackTheMetadata({
customer_id: payload.customer_id
})
const {
user
} = await this.getUser(
payload.user_id,
metadata
);
const userDto: UserDTO = {
id: user.id,
name: user.username,
jobTitle: user?.jobTitle || null,
department: user?.department || null,
hierarchy: user?.hierarchy || null,
customer: {
id: payload.customer_id,
name: payload.customer_name,
tier: payload.customer_tier,
}
};
return userDto;
}
}
+13
View File
@@ -140,3 +140,16 @@ export interface BulkEditResponse {
successfulUsers: string[];
failedUsers: string[];
}
export type UserDTO = {
id: string,
name: string,
jobTitle?: string,
department?: string,
hierarchy?: string,
customer: {
id: string,
name: string,
tier: string,
}
}
+9 -4
View File
@@ -197,9 +197,11 @@ class CatalogService implements OnModuleInit {
async getUserRolesIds(userId: string) {
const result = await this.userService.findOneById(userId).catch(() => null);
const roles_ids = result.user.roles.map((role) => role.id);
if (result) {
return result.user.roles.map((role) => role.id);
}
return roles_ids;
return [];
}
async searchDataAssets(
@@ -406,11 +408,14 @@ class CatalogService implements OnModuleInit {
const roles = [];
const users = [];
for (const role_id of data_asset.roles) {
const data_asset_roles = data_asset?.roles || []
for (const role_id of data_asset_roles) {
const role = customer_roles.find((r) => r.id === role_id);
if (role) roles.push({ id: role.id, name: role.name });
}
for (const user_id of data_asset.users) {
const data_asset_users = data_asset?.users || []
for (const user_id of data_asset_users) {
const user = customer_users.find((r) => r.id === user_id);
if (user) users.push({ id: user.id, username: user.username });
}
+25 -8
View File
@@ -1,29 +1,46 @@
import { Body, Controller, Inject, Param, Post, Req } from '@nestjs/common';
import { init } from 'mixpanel';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { RequestUser } from 'src/decorators/user.decorator';
import { MixpanelService } from './mixpanel.service';
import { extractUserFrom } from 'src/authentication/extract-user';
import DadosferaLogger from '@dadosfera/dadosfera-logs';
@ApiInternalOnlyController()
@Controller('trackEvent')
export class MixpanelController {
logger: DadosferaLogger;
constructor(
private mixpanelService: MixpanelService
) {}
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private mixpanelService: MixpanelService,
) {
this.logger = dadosferaLogger.logger;
}
@Post(':id')
async trackEvent(
@Param('id') id,
@Body() body,
@User() user: RequestUser,
@Req() request
) {
this.logger.info(`POST Track Event: ${id}`)
delete body.info;
const anonymousUser = {
username: "anonymous",
customer_name: "anonymous"
} as RequestUser
const hasToken = request.headers['authorization'];
const user = hasToken ? extractUserFrom(hasToken) : anonymousUser;
this.logger.info(`Has user: ${typeof hasToken == "string"}`)
await this.mixpanelService.track(id, user, request, body)
this.logger.info(`Event successful`)
return { id, body, user: user.username };
}
}
+3 -20
View File
@@ -3,6 +3,7 @@ import { ApiOperation, ApiTags } from '@nestjs/swagger';
import {
AuthenticateCondition,
Authenticated,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
@@ -13,26 +14,6 @@ import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiTags('Pipelines')
@Controller('pipelines')
@Authenticated()
@AuthenticateCondition((req, user) => {
let action;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
default:
action = req.method;
}
return user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions[action].seqid,
);
})
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@@ -44,6 +25,7 @@ export class PipelinesController {
}
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiOperation({
deprecated: true,
description:
@@ -71,6 +53,7 @@ export class PipelinesController {
description:
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
+21 -34
View File
@@ -26,6 +26,7 @@ import {
import {
AuthenticateCondition,
RequireAllPermissions,
RequireSomePermission,
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
@@ -52,30 +53,6 @@ import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('pipelinesV2')
@AuthenticateCondition((req, user) => {
let action;
switch (req.method) {
case 'POST':
action = 'CREATE';
break;
case 'PUT':
action = 'UPDATE';
break;
case 'PATCH':
action = 'UPDATE';
break;
default:
action = req.method;
}
return user.permissions.includes(
PERMISSIONS_GROUPS.PIPELINE.permissions[action].seqid,
);
})
export class PipelinesController {
logger: DadosferaLogger;
constructor(
@@ -88,6 +65,7 @@ export class PipelinesController {
}
@Get('monitoring-dashboard')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getMonitoringDashboard(@User() user: RequestUser) {
this.logger.info('PipelinesController - getMonitoringDashboard', { user });
@@ -100,6 +78,7 @@ export class PipelinesController {
}
@Post()
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@ApiCreatedResponse({ type: IPipelineV2 })
async create(
@Language() language: LanguageEnum,
@@ -126,6 +105,7 @@ export class PipelinesController {
}
@Get()
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async findAll(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -150,6 +130,7 @@ export class PipelinesController {
}
@Get('/download-logs')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async downloadLogs(
@User() user: RequestUser,
@Language() language: LanguageEnum,
@@ -180,6 +161,7 @@ export class PipelinesController {
}
@Get(':id/config')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW,PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineproperties(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@@ -191,6 +173,7 @@ export class PipelinesController {
}
@Get(':id/objects')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineObjects(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@@ -202,16 +185,14 @@ export class PipelinesController {
}
@Get(':id/status')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
this.logger.info(`/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
});
const response = await this.oldPipelinesService.getPipelineStatus(body);
@@ -219,6 +200,7 @@ export class PipelinesController {
}
@Get('/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.GET)
async findOne(
@Language() language: LanguageEnum,
@User() user: RequestUser,
@@ -256,10 +238,12 @@ export class PipelinesController {
});
return res;
});
return result;
}
@Patch('/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async update(
@Language() language: LanguageEnum,
@Body() updatePipelineDto,
@@ -299,6 +283,7 @@ export class PipelinesController {
deprecated: true,
description: 'This method is deprecated. Please use PATCH instead',
})
@RequireSomePermission(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW, PERMISSIONS_GROUPS.PIPELINE.permissions.UPDATE)
async updateDeprecated(
@Language() language: LanguageEnum,
@Body() updatePipelineDto,
@@ -314,6 +299,7 @@ export class PipelinesController {
@Delete(':id')
@ApiNoContentResponse()
@HttpCode(HttpStatus.NO_CONTENT)
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE)
async delete(@Param('id') id: string, @User() user: RequestUser) {
this.logger.info('PipelinesController - delete', { user });
const metadata = PackTheMetadata({
@@ -327,7 +313,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('/init-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
async initUploadFile(
@User() user: RequestUser,
@Body() body: IInitUploadCSVFile,
@@ -359,7 +345,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('/complete-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
async completeUploadFile(
@User() user: RequestUser,
@Body() body: ICompleteUploadCSVFile,
@@ -377,7 +363,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('/file')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
@RequireAllPermissions(PERMISSIONS_GROUPS.IMPORT_FILES.permissions.VIEW)
async uploadedFile(
@User() user: RequestUser,
@Body() body: ICreatePipelineCSVFile,
@@ -427,6 +413,7 @@ export class PipelinesController {
@ApiInternalOnlyEndpoint()
@Post('start/:id')
@RequireSomePermission(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async activate(@Param('id') id: string, @Body() body) {
const { info } = body;
+20
View File
@@ -124,4 +124,24 @@ export class ThemeController {
}
}
@Post('/:id/theme/reset')
@ApiOkResponse({ type: CustomerThemeResponse })
async resetTheme(@Param('id') id: string) {
this.logger.info('getCustomerTheme with id' + id);
try {
await this.themeService.resetTheme(id);
return { theme: null };
}catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND) {
this.logger.error('Error - getCustomerTheme - Expect CUSTOMER.NOT_FOUND');
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
} else {
this.logger.error('Error - getCustomerTheme Unknown Error:' + err?.message);
return { theme: null };
};
}
}
}
+12
View File
@@ -44,6 +44,18 @@ export class ThemeService implements OnModuleInit {
);
}
async resetTheme(id: string) {
const { theme } = await firstValueFrom(
this.themeService.ResetCustomerTheme({
id
}),
);
return {
theme
}
}
async createThemeByCustomer(id: string, theme: CustomerThemeRequest & Files) {
if (!id) {
this.logger.error('Error - saveCustomertheme - not found id:' + id);
+14 -4
View File
@@ -5,15 +5,25 @@ import { redisStore } from 'cache-manager-ioredis-yet';
@Module({
imports: [
CacheModule.registerAsync({
useFactory: async () => ({
store: await redisStore({
useFactory: async () => {
const baseRedisConfig = {
ttl: 5 * 1000 * 60, // 5 minute
host: process.env.REDIS_HOST,
port: process.env.REDIS_PORT && Number(process.env.REDIS_PORT),
db: process.env.REDIS_DATABASE && Number(process.env.REDIS_DATABASE),
keyPrefix: 'maestro:sso',
}),
}),
}
if (process.env.ENV !== 'local') {
baseRedisConfig['tls'] = {
servername: process.env.REDIS_HOST,
}
}
return {
store: await redisStore(baseRedisConfig),
}
},
}),
],
providers: [CacheService],