Compare commits

...
59 Commits
Author SHA1 Message Date
Gabriel Amorim 44bb205483 Merge pull request #214 from dadosfera/fix/get-links-permission
FIX: remove permission check to get customer links
2023-05-05 13:52:38 -03:00
Gabriel Amorim edb4622684 FIX: remove permission to get customer links 2023-05-05 12:13:55 -03:00
Victor Radael abb46d78da Merge pull request #212 from dadosfera/feature/customer-links
SOLENG-1481 - Customer links endpoints
2023-04-17 17:01:41 -03:00
Victor Radael 04fdc0368a Merge pull request #213 from dadosfera/update/hubspot-tables
UPDATE: HubspotTables
2023-04-15 14:49:47 -03:00
Victor Radael 50d76622c5 UPDATE: HubspotTables 2023-04-15 14:45:18 -03:00
Victor Radael 238f56cb76 Merge branch 'SOLENG-1736' into feature/customer-links 2023-04-14 14:44:57 -03:00
Victor Radael e721d6f5f5 Merge branch 'main' into feature/customer-links 2023-04-14 13:30:57 -03:00
Victor Radael 44df13e820 UPDATE: Att protospack-v2 version 2023-04-14 13:13:47 -03:00
Victor Radael 1fd0f79dd5 Att hubspot connection 2023-04-13 15:41:44 -03:00
Victor Radael 1b92371737 Att hubspot connection 2023-04-13 14:55:33 -03:00
Colombo bf5b3484a1 FEAT: add controller method interfaces 2023-04-11 20:39:47 -03:00
Colombo eb1e248d4b FEAT: add customer (links) endpoint 2023-04-11 17:04:40 -03:00
Colombo 0827414051 FEAT: add new dto property 2023-04-10 19:36:36 -03:00
Colombo 081986607b CHORE: change protospack package to beta 2023-04-06 17:07:23 -03:00
Gabriel Amorim 189080199a Merge pull request #211 from dadosfera/feat/trigger-catalog-task
Feat/trigger catalog task
2023-03-17 12:14:15 -03:00
Gabriel Amorim 55281e3965 FIX: docs 2023-03-16 17:58:11 -03:00
Gabriel Amorim db8a6175e8 new protospack version 2023-03-16 17:45:15 -03:00
Gabriel Amorim 7b0d4dd5bc FIX: set route params type 2023-03-16 17:45:03 -03:00
Gabriel Amorim 6fd277ef3a docs 2023-03-14 16:02:22 -03:00
Gabriel Amorim 7b25bfdb69 FIX: add table_schema to request 2023-03-14 16:00:57 -03:00
Gabriel Amorim 2abc19c589 FEAT: trigger and get dataset cataloging task 2023-03-14 13:39:21 -03:00
Gabriel Amorim faf6b1df75 Merge pull request #210 from dadosfera/fix/delete-connector
Fix/delete connector
2023-03-09 18:14:14 -03:00
Gabriel Amorim 422be4a422 swaggers 2023-03-06 12:11:13 -03:00
Gabriel Amorim da136e0ef0 FIX: better swagger generation strategy (Only PRD server on external docs) 2023-03-06 12:10:12 -03:00
Gabriel Amorim 737cc9b8ea FIX: removed wrong JSON.parse on connector delete response 2023-03-06 12:09:29 -03:00
Gabriel Amorim 23f6b179fd Merge pull request #209 from dadosfera/feat/google-sheets-link
Feat/google sheets link
2023-02-27 16:03:52 -03:00
Gabriel Amorim 03e451c21d Merge branch 'main' into feat/google-sheets-link 2023-02-27 11:26:03 -03:00
Gabriel Amorim be061a0fb9 Merge pull request #208 from dadosfera/fix/upload-files
FIX: create pipeline from any file type on upload
2023-02-24 17:10:08 -03:00
Gabriel Amorim c83f6a68d1 update protospack version 2023-02-24 16:50:29 -03:00
Gabriel Amorim 2646e3e4f6 FEAT: return google sheets entire link 2023-02-24 15:58:06 -03:00
Gabriel Amorim 06424d6bcf FIX: docs 2023-02-24 14:47:49 -03:00
Gabriel Amorim 45b82515fc FIX: do not return config_controls that the pipeline don't have 2023-02-24 14:31:50 -03:00
Gabriel Amorim a9cc951e34 FIX: new error codes 2023-02-23 17:14:47 -03:00
Gabriel Amorim 93ccf506fb FIX: csv import with header=false 2023-02-22 15:13:36 -03:00
Gabriel Amorim 706d0ccfa0 FIX: get right image url 2023-02-22 14:32:41 -03:00
Beatriz Antunes c2eec4e95c Merge pull request #207 from dadosfera/fix/hubspot-chat-permission
FIX: change permission to access support chat
2023-02-22 13:10:54 -03:00
Gabriel Amorim ac3132b6eb FIX: create pipeline from any file type on upload 2023-02-22 10:18:28 -03:00
Gabriel Amorim ee8c5e7a93 Merge pull request #206 from dadosfera/feat/docs-refactor
Feat: docs refactor
2023-02-17 14:36:19 -03:00
Gabriel Rosa a2c1f2da6f FIX: new zendesk permission 2023-02-17 14:33:14 -03:00
Gabriel Rosa 7ecb50315f Update action 2023-02-17 11:01:29 -03:00
Gabriel Rosa 84b3370826 Update action 2023-02-17 11:01:08 -03:00
Gabriel Rosa aa9e0536a7 FIX: tests 2023-02-17 09:55:10 -03:00
Gabriel Rosa 544d13fac5 FIX: readme improved 2023-02-16 17:01:49 -03:00
Gabriel Rosa a4ff07a285 FIX: removed connection test from external docs 2023-02-16 12:07:47 -03:00
Gabriel Rosa 28578d3aa8 FIX: better query params on catalog get all 2023-02-15 16:46:32 -03:00
Gabriel Rosa b8ea729a33 FIX: add external docs to docsfera 2023-02-15 14:16:28 -03:00
Gabriel Rosa b383070b68 Internal docs 2023-02-15 14:07:26 -03:00
Gabriel Rosa 87882b5afc FIX: add multiple servers 2023-02-15 13:14:53 -03:00
Gabriel Rosa 6a14c2b459 FIX: hide internal endpoints 2023-02-15 13:13:43 -03:00
Gabriel Rosa 99288fa805 FIX: new sematic release version 2023-02-14 16:55:39 -03:00
Gabriel Rosa 35cc0094b3 FIX: new sematic release 2023-02-14 16:51:53 -03:00
Gabriel Rosa 98262bcba9 FIX: Run semantic release on manual deploy 2023-02-14 16:48:46 -03:00
Gabriel Rosa a2c9a462e4 Run semantic release on manual deploy 2023-02-14 16:46:54 -03:00
Gabriel Rosa c5c089e5d5 Better error handling and documentation on refresh token 2023-02-14 09:25:14 -03:00
Gabriel Rosa de06512249 FIX: changed language strategy to use new custom decorator 2023-02-14 08:56:23 -03:00
Gabriel Rosa 9058b80383 Merge branch 'main' into feat/docs-refactor 2023-02-13 12:16:12 -03:00
Gabriel Rosa fbc0d2eede FIX: moved decorators to a separate folder 2023-02-10 14:23:27 -03:00
Gabriel Rosa 9fcbeda470 FIX: deprecating old pipeline routes 2023-02-08 18:07:06 -03:00
Gabriel Rosa 8043635b1c FIX: add authentication to swagger 2023-02-08 15:23:49 -03:00
55 changed files with 7560 additions and 2060 deletions
+19 -2
View File
@@ -76,7 +76,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Update Pip
run: |
@@ -146,7 +146,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Extract Docs BlockId and PageId
env:
@@ -187,3 +187,20 @@ jobs:
sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.json
sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.json
curl -X POST -H 'Content-Type: application/json' -d @docsfera.json $DOCS_URL
- name: Generate External API docs
env:
DOCS_URL: ${{ secrets.DOCS_URL }}
DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }}
DOCS_PAGE_ID: ${{secrets.EXTERNAL_DOCS_PAGE_ID}}
DOCS_BLOCK_ID: ${{secrets.EXTERNAL_DOCS_BLOCK_ID}}
EVENT: ${{ github.event_name }}
RELEASE_VERSION: ${{ needs.semantic_release.outputs.new_release_version }}
run: |
if [ ${EVENT} != "workflow_dispatch" ]; then
sed -i -E "s/(\"title\": )\"Maestro.+\"/\1\"Maestro - $RELEASE_VERSION\"/g" docsfera.external.json
fi
sed -i -e "s/\${DOCS_API_TOKEN}/$DOCS_API_TOKEN/g" docsfera.external.json
sed -i -e "s/\${DOCS_PAGE_ID}/$DOCS_PAGE_ID/g" docsfera.external.json
sed -i -e "s/\${DOCS_BLOCK_ID}/$DOCS_BLOCK_ID/g" docsfera.external.json
curl -X POST -H 'Content-Type: application/json' -d @docsfera.external.json $DOCS_URL
+1 -1
View File
@@ -9,7 +9,7 @@ jobs:
runs-on: self-hosted
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Build
env:
+120 -71
View File
@@ -1,67 +1,98 @@
<p align="center">
<image src="./assets/maestro.svg" style="width:10rem">
</p>
</p>
# Maestro
Maestro is the Dadosfera's gateway, it's responsible for the communication between the frontend application and Dadosfera's microservices.
## 🚀 Starting
These instructions will allow you to get a working copy of the project on your local machine for development and testing purposes.
### 📋 Requirements
- [NodeJS v18.3.0 LTS / NPM v8.11](https://nodejs.org/pt-br/download/) (you can opt to use [NVM](https://github.com/nvm-sh/nvm) to easily manage node versions)
- Request access to AWS Console dev account for **all services** (avoid gradually asking for each needed service. it will slow down your development cycle)
- Create your Access Key on the "Security credentials" menu
- Set the Access Key on your local development machine
- Create your Access Key on the "Security credentials" menu
- Set the Access Key on your local development machine
- Request access to the dev, stg and prd VPNs
### 🔧 Installation<a id="installation"></a>
- Clone the repository
- SSH
```
git clone git@github.com:dadosfera/maestro.git
```
or
- HTTPS
```
git clone https://github.com/dadosfera/maestro.git
```
- Clone the repository
- SSH
```
git clone git@github.com:dadosfera/maestro.git
```
or
- HTTPS
```
git clone https://github.com/dadosfera/maestro.git
```
- Select the correct node version (optional, only if using [NVM](https://github.com/nvm-sh/nvm)):
```sh
nvm use
```
```sh
nvm use
```
- Install the project dependencies:
```sh
npm i
```
```sh
npm i
```
- Setup the following enviroment variables:
```
ENV=
DUC_URL=
INFACTORY_URL=
OTFACTORY_URL=
PIFACTORY_URL=
SM_OAUTH_PATH=
```
```
ENV=
DUC_URL=
INFACTORY_URL=
OTFACTORY_URL=
PIFACTORY_URL=
SM_OAUTH_PATH=
```
- Start the server:
```sh
# dev mode
npm run start:dev
# or in debug mode
npm run start:debug
```
The service should start successfully.
```sh
# dev mode
npm run start:dev
# or in debug mode
npm run start:debug
```
The service should start successfully.
## 📄 Documentation
NestJs makes it easy to document each route using decorators on all requests and responses properties. It automatically generates a swagger for given information and provides a route to access it http://localhost:3333/api.
For more info check the [official documentation](https://docs.nestjs.com/openapi/introduction).
### - Multiple documentations
We are currently generating **2 different** documentations: Internal and External.
All routes that have the decorator `@ApiInternalOnly()` will not be visible on the **External** API swagger.
- When you start the application with `npm run start` it will serve and generate the swagger JSON of the **External** API
- When you start the application with `npm run start:internal` it will serve and generate the swagger JSON of the **Internal** API
- When you run `npm run docs` it will generate the swagger JSON of both **Internal** and **External** API, and save them to `docsfera.json` and `docsfera.external.json` respectively;
It is important to run `npm run docs` before every deploy so we can always have the most updated docs published.
## Authentication decorators
Maestro have utilities to ease the user authentication on every controller and route. The following decorators are available:
### `@Authenticated`
If the user must be authenticated to make request, we can use the `@Authenticated` decorator in the controller or route, as needed.
```ts
@@ -106,6 +137,7 @@ class FooController {
```
### `@RequireAllPermissions`
This decorator requires that **all permissions** listed are granted to the requesting user.
```ts
@@ -125,6 +157,7 @@ class FooController {
```
### `@RequireSomePermission`
In the following case, the user is required to have **at least one** listed permission.
```ts
@@ -144,9 +177,11 @@ class FooController {
```
### `@AuthenticateCondition`
If a more complicated authentication check needs to be done, we can use the `@AuthenticateCondition` decorator to define it. The custom function must return `true` to authenticate the request.
In the following example:
- all routes on the `FooController` controller can only be requested from localhost
- `POST /foo/bar` can only be requested from localhost **and** by users from customer id `111...eef`
@@ -166,7 +201,10 @@ class FooController {
@Post('bar')
// allow requests only from a specific customer
@AuthenticateCondition((request: Request, user: RequestUser) => user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef')
@AuthenticateCondition(
(request: Request, user: RequestUser) =>
user.customer_id === '1113e943-2187-4fdd-9c2c-54338fedaeef',
)
async postBar() {
/* ... */
}
@@ -174,51 +212,55 @@ class FooController {
```
### Note on authentication decorators
- The old authentication method placed the user data in the `request.body.info` field, this imposes certain issues regarding the request body because this data should be from the frontend without any modification by Maestro. Now this usage is ⚠️ **DEPRECATED** ⚠️. We are working to migrate to the `@User` parameter decorator. The old method is working while the migration is in progress.
- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (*and wouldn't like to...*) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
- Authentication decorators can be used together and all of them **must** pass to the request be authenticated, but in the general case you don't need to (_and wouldn't like to..._) use all of them together, as you can code all of the authentication logic in the `@AuthenticateCondition` decorator.
```ts
import {
RequireAllPermissions,
RequireSomePermission,
AuthenticateCondition,
} from '../../authentication/authentication.decorator';
import { PERMISSIONS } from '../../authentication/permissions.enum';
import { Waa, Baz } from './authenticationFunctions'
```ts
import {
RequireAllPermissions,
RequireSomePermission,
AuthenticateCondition,
} from '../../authentication/authentication.decorator';
import { PERMISSIONS } from '../../authentication/permissions.enum';
import { Waa, Baz } from './authenticationFunctions';
@Controller('foo')
@RequireAllPermissions(Permissions.FOO.USE, Permissions.FOO.REQUEST)
@RequireSomePermission(Permissions.FOO.MANAGE, Permissions.FOO.ADMIN)
@AuthenticateCondition(Waa)
@AuthenticateCondition(Baz)
class FooController {
@Controller('foo')
@RequireAllPermissions(Permissions.FOO.USE, Permissions.FOO.REQUEST)
@RequireSomePermission(Permissions.FOO.MANAGE, Permissions.FOO.ADMIN)
@AuthenticateCondition(Waa)
@AuthenticateCondition(Baz)
class FooController {
/* ... */
@Get('bar')
async getBar() {
/* ... */
@Get('bar')
async getBar() {
/* ... */
}
@Post('bar')
@RequireAllPermissions(Permissions.BAR.MANAGE, Permissions.BAR.USE, Permissions.BAR.REQUEST)
async postBar() {
/* ... */
}
}
```
@Post('bar')
@RequireAllPermissions(
Permissions.BAR.MANAGE,
Permissions.BAR.USE,
Permissions.BAR.REQUEST,
)
async postBar() {
/* ... */
}
}
```
- If `@RequireAllPermissions` and `@RequireSomePermission` are used with **only a single permission**, they present the **exactly same behavior**.
```ts
// same behavior
@RequireAllPermissions(Permissions.BAR.MANAGE)
@RequireSomePermission(Permissions.BAR.MANAGE)
```
```ts
// same behavior
@RequireAllPermissions(Permissions.BAR.MANAGE)
@RequireSomePermission(Permissions.BAR.MANAGE)
```
## `@User` parameter decorator
The requesting user data can be obtained using the @User parameter decorator, like in the following snippet:
```ts
@@ -264,12 +306,15 @@ class UserController {
```
## 📦 Development
### ⌨️ Coding Style
By default, we use [ESLint](https://eslint.org/) + [Prettier](https://prettier.io/) with default settings.
**We recommend using Visual Studio Code and installing the recommended extensions to ease the development process.**
### Commits pattern
Our workflow pipeline follows the conventional commits specs ([cheat sheets](https://cheatography.com/albelop/cheat-sheets/conventional-commits/)) to release versions accordingly.
Format: `<type>[optional scope]: <description>`
@@ -277,25 +322,29 @@ Format: `<type>[optional scope]: <description>`
Example: `FIX: ensure Range headers adhere more closely to RFC 2616`
### Branching naming convention
- **Feature**: Any code changes for a new module or use case should be done on a feature branch. This branch is created based on the `main` branch. When all changes are done, a Pull Request/Merge Request is needed to put all of these changes back to the `main` branch. Examples: `feature/integrate-swagger`, `feature/JIRA-1234`, `feature/JIRA-1234_support-dark-theme`.
**It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (_) could be used to separate the ID and description.**
**It is recommended to use all lower caps letters and hyphen (-) to separate words unless it is a specific item name or ID. Underscore (\_) could be used to separate the ID and description.**
- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
- **Bug Fix**: If the code changes made from the feature branch were rejected after a release, sprint or demo, any necessary fixes after that should be done on the bugfix branch. Examples: `bugfix/more-gray-shades`, `bugfix/JIRA-1444_gray-on-blur-fix`.
- **Hot Fix**: If there is a need to fix a blocker, do a temporary patch, apply a critical framework or configuration change that should be handled immediately, it should be created as a Hotfix. Examples: `hotfix/disable-endpoint-zero-day-exploit`, `hotfix/increase-scaling-threshold`.
- **Experimental**: A branch for playing around. Any new feature or idea that is not part of a release or a sprint. Example: `experimental/dark-theme-support`.
### Making a Pull Request
1. Commit your changes
2. Open the Pull Request on GitHub
3. Send Pull Request link in Microsfera Google Chat Group for review and possible approval
## 🛠️ Built with
Some technologies used in this project:
- [NestJS](https://docs.nestjs.com) - Framework for building efficient and scalable NodeJS server-side applications
## ⚙️ Back-end Architecture
The architecture can be found at [this link](https://sites.google.com/dadosfera.ai/wikidoproduto/time/back-end).
File diff suppressed because it is too large Load Diff
+3348 -1630
View File
File diff suppressed because it is too large Load Diff
+18
View File
@@ -0,0 +1,18 @@
declare global {
namespace NodeJS {
interface ProcessEnv {
NODE_ENV: 'test';
ENV: 'local' | 'stg' | 'prd' | 'test';
LOCAL_ENV: 'stg' | 'prd';
DUC_URL: string;
INFACTORY_URL: string;
PIFACTORY_URL: string;
INTERNAL_SWAGGER: 'true' | 'false';
AWS_REGION: string;
}
}
}
export {};
+32 -23
View File
@@ -12,7 +12,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.29.0",
"@dadosfera/protospack-v2": "3.30.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
@@ -1727,9 +1727,9 @@
}
},
"node_modules/@dadosfera/protospack-v2": {
"version": "3.29.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz",
"integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==",
"version": "3.30.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.30.0.tgz",
"integrity": "sha512-00wqZyZmjd6rMECga2S3Ow0dk6c5fZoWd6Th+Ajofq6q/kycTU66p/5xdf0sPf//WyXMGCXasqGSZa0h9i01vg==",
"dependencies": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -4495,9 +4495,10 @@
"license": "MIT"
},
"node_modules/cookiejar": {
"version": "2.1.3",
"dev": true,
"license": "MIT"
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz",
"integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==",
"dev": true
},
"node_modules/core-util-is": {
"version": "1.0.3",
@@ -7759,9 +7760,10 @@
"license": "ISC"
},
"node_modules/json5": {
"version": "2.2.1",
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
"dev": true,
"license": "MIT",
"bin": {
"json5": "lib/cli.js"
},
@@ -7968,9 +7970,9 @@
}
},
"node_modules/luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==",
"version": "1.28.1",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.1.tgz",
"integrity": "sha512-gYHAa180mKrNIUJCbwpmD0aTu9kV0dREDrwNnuyFAsO1Wt0EVYSZelPnJlbj9HplzXX/YWXHFTL45kvZ53M0pw==",
"engines": {
"node": "*"
}
@@ -10384,9 +10386,10 @@
}
},
"node_modules/tsconfig-paths/node_modules/json5": {
"version": "1.0.1",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz",
"integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"minimist": "^1.2.0"
},
@@ -12284,9 +12287,9 @@
}
},
"@dadosfera/protospack-v2": {
"version": "3.29.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.29.0.tgz",
"integrity": "sha512-G66u9V+/+5rQb+fpJdDXydTaji2Mlr0Ro5TTYcQleITKU9DjF1WZs/jP/Qu0Y224nCb8HhePvaVl/Rr0wtLt4w==",
"version": "3.30.0",
"resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com:443/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.30.0.tgz",
"integrity": "sha512-00wqZyZmjd6rMECga2S3Ow0dk6c5fZoWd6Th+Ajofq6q/kycTU66p/5xdf0sPf//WyXMGCXasqGSZa0h9i01vg==",
"requires": {
"@grpc/grpc-js": "^1.6.7",
"rxjs": "^7.5.5",
@@ -14190,7 +14193,9 @@
"version": "1.0.6"
},
"cookiejar": {
"version": "2.1.3",
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz",
"integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==",
"dev": true
},
"core-util-is": {
@@ -16399,7 +16404,9 @@
"dev": true
},
"json5": {
"version": "2.2.1",
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
"dev": true
},
"jsonc-parser": {
@@ -16548,9 +16555,9 @@
}
},
"luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
"version": "1.28.1",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.1.tgz",
"integrity": "sha512-gYHAa180mKrNIUJCbwpmD0aTu9kV0dREDrwNnuyFAsO1Wt0EVYSZelPnJlbj9HplzXX/YWXHFTL45kvZ53M0pw=="
},
"macos-release": {
"version": "2.5.0",
@@ -18138,7 +18145,9 @@
},
"dependencies": {
"json5": {
"version": "1.0.1",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz",
"integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==",
"dev": true,
"requires": {
"minimist": "^1.2.0"
+5 -4
View File
@@ -15,9 +15,10 @@
"build": "nest build",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
"start": "nest start",
"start:dev": "nest start --watch",
"start:debug": "nest start --debug --watch",
"start:prod": "npm run build && node dist/main",
"start:dev": "export INTERNAL_SWAGGER=true && nest start --watch",
"start:debug": "npm run start:dev",
"start:prod": "node dist/main",
"docs": "export KILL_AFTER_START=1 && nest start && export INTERNAL_SWAGGER=true && nest start",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest",
"test:watch": "jest --watch",
@@ -28,7 +29,7 @@
"dependencies": {
"@aws-sdk/client-secrets-manager": "^3.112.0",
"@dadosfera/dadosfera-logs": "^1.0.0-beta.4",
"@dadosfera/protospack-v2": "3.29.0",
"@dadosfera/protospack-v2": "3.31.0",
"@grpc/grpc-js": "^1.6.7",
"@grpc/proto-loader": "^0.6.13",
"@nestjs/common": "^8.4.7",
+9 -7
View File
@@ -25,6 +25,7 @@ import { ConfigModule } from '@nestjs/config';
import { PipelinesV2Module } from './modules/pipelinesV2/pipelines.module';
import { ProductboardModule } from './modules/productboard/productboard.module';
import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
import { CustomersModule } from './modules/customers/customers.module';
@Module({
controllers: [],
@@ -40,24 +41,25 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module';
isGlobal: true,
}),
AuthModule,
ConnectionModule,
NetworkConfigModule,
InputsModule,
PipelinesV2Module,
CatalogModule,
ConnectorModule,
PermissionsModule,
TermsOfUseModule,
ConnectionModule,
NetworkConfigModule,
ConnectionTestModule,
PipelinesModule,
PipelinesV2Module,
TransformationsModule,
HealthModule,
CatalogModule,
UsersModule,
RolesModule,
InputsModule,
OauthModule,
CatalogModule,
ProductboardModule,
MixpanelModule,
CustomersModule,
//Always leave HealthModule last, so it is on the bottom of swagger
HealthModule,
],
})
export class AppModule {}
@@ -1,56 +0,0 @@
import { Request } from 'express';
import { CustomDecorator } from '@nestjs/common';
import { PermissionsObjectPermission } from './permissions.enum';
import { RequestUser } from './user.decorator';
export const AUTH_FUNCTION_KEY = '__AUTH_FUNCTION__';
export type AuthenticationFunction = (req: Request, user: any) => boolean;
// implementation copied from SetMetadata, but tweaked to get existing values
// of the metadataKey and accumulate it with the new metadataValue
function SetMultipleMetadata(metadataKey, metadataValue): CustomDecorator {
const decoratorFactory = (target, key, descriptor) => {
// .start: tweak
// descriptor?.value = function decorator; target = class decorator
const accumulatedVal =
Reflect.getMetadata(metadataKey, descriptor?.value ?? target) ?? [];
accumulatedVal.push(metadataValue);
// .end: tweak
if (descriptor) {
Reflect.defineMetadata(metadataKey, accumulatedVal, descriptor.value);
return descriptor;
}
Reflect.defineMetadata(metadataKey, accumulatedVal, target);
return target;
};
decoratorFactory.KEY = metadataKey;
return decoratorFactory as any;
}
export function RequireAllPermissions(
...permissions: PermissionsObjectPermission[]
) {
return SetMultipleMetadata(AUTH_FUNCTION_KEY, (req, user: RequestUser) =>
permissions.every(({ seqid }) => user.permissions.includes(seqid)),
);
}
export function RequireSomePermission(
...permissions: PermissionsObjectPermission[]
) {
return SetMultipleMetadata(AUTH_FUNCTION_KEY, (req, user: RequestUser) =>
permissions.some(({ seqid }) => user.permissions.includes(seqid)),
);
}
export function AuthenticateCondition(func: AuthenticationFunction) {
return SetMultipleMetadata(AUTH_FUNCTION_KEY, func);
}
export function Authenticated() {
return SetMultipleMetadata(AUTH_FUNCTION_KEY, () => true);
}
@@ -11,7 +11,7 @@ import {
Authenticated,
RequireAllPermissions,
RequireSomePermission,
} from './authentication.decorator';
} from '../decorators/authentication.decorator';
import { AuthenticationGuard } from './authentication.guard';
import { PERMISSIONS_GROUPS } from './permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
+2 -2
View File
@@ -13,8 +13,8 @@ import { AuthClientService } from '../modules/auth/auth.service';
import {
AuthenticationFunction,
AUTH_FUNCTION_KEY,
} from './authentication.decorator';
import { RequestUser } from './user.decorator';
} from '../decorators/authentication.decorator';
import { RequestUser } from '../decorators/user.decorator';
import ErrorBuilder from '../utils/ErrorBuilder';
import ErrorCodes from '../utils/errorCodes';
+14 -4
View File
@@ -346,6 +346,16 @@ export const PERMISSIONS_GROUPS = {
'es-es': 'Acceder al Módulo de Incorporación de Activos',
},
},
TRIGGER_CATALOG_TASK: {
seqid: 45,
claim: 'catalog:trigger-task',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Executar a catalogação de um ativo',
'en-us': 'Trigger an asset cataloging',
'es-es': 'Realizar el listado de un activo',
},
},
},
},
@@ -428,12 +438,12 @@ export const PERMISSIONS_GROUPS = {
permissions: {
OPEN: {
seqid: 16,
claim: 'GET /zendesk',
claim: 'support-chat:access',
usage: PermissionUsages.INTERNAL,
name: {
'pt-br': 'Acessar Zendesk',
'en-us': 'Access Zendesk',
'es-es': 'Acceso Zendesk',
'pt-br': 'Acessar chat de suporte',
'en-us': 'Access support chat',
'es-es': 'Acceder al chat de soporte',
},
},
},
@@ -0,0 +1,41 @@
import { applyDecorators } from '@nestjs/common';
import { ApiSecurity } from '@nestjs/swagger';
import { type Request } from 'express';
import { type PermissionsObjectPermission } from '../authentication/permissions.enum';
import { type RequestUser } from './user.decorator';
import { SetMultipleMetadata } from './shared';
export const AUTH_FUNCTION_KEY = '__AUTH_FUNCTION__';
export type AuthenticationFunction = (req: Request, user: any) => boolean;
export function RequireAllPermissions(
...permissions: PermissionsObjectPermission[]
) {
return createAuthenticatedDecorator((req, user: RequestUser) =>
permissions.every(({ seqid }) => user.permissions.includes(seqid)),
);
}
export function RequireSomePermission(
...permissions: PermissionsObjectPermission[]
) {
return createAuthenticatedDecorator((req, user: RequestUser) =>
permissions.some(({ seqid }) => user.permissions.includes(seqid)),
);
}
export function AuthenticateCondition(func: AuthenticationFunction) {
return createAuthenticatedDecorator(func);
}
export function Authenticated() {
return createAuthenticatedDecorator(() => true);
}
function createAuthenticatedDecorator(metadataValue: AuthenticationFunction) {
return applyDecorators(
ApiSecurity('access-token'),
SetMultipleMetadata(AUTH_FUNCTION_KEY, metadataValue),
);
}
+25
View File
@@ -0,0 +1,25 @@
import { createParamDecorator, type ExecutionContext } from '@nestjs/common';
import { LanguageEnum } from 'src/utils/languages.enum';
export const Language: () => ParameterDecorator = createParamDecorator(
(options: any, ctx: ExecutionContext): LanguageEnum => {
const headers = ctx.switchToHttp()?.getRequest()?.headers;
if (!headers) return LanguageEnum.ptbr;
let language: LanguageEnum =
headers['dadosfera-lang'] ?? headers['accept-language'];
language = language?.toLowerCase().trim() as LanguageEnum;
if (!language) return LanguageEnum.ptbr;
Object.values(LanguageEnum).some((l) => {
if (language === l || l.includes(language)) {
language = l;
return true;
}
language = LanguageEnum.enus;
});
return language;
},
);
+28
View File
@@ -0,0 +1,28 @@
import { type CustomDecorator } from '@nestjs/common';
// implementation copied from SetMetadata, but tweaked to get existing values
// of the metadataKey and accumulate it with the new metadataValue
export function SetMultipleMetadata(
metadataKey,
metadataValue,
): CustomDecorator {
const decoratorFactory: CustomDecorator = (target, key?, descriptor?) => {
// .start: tweak
// descriptor?.value = function decorator; target = class decorator
const accumulatedVal =
Reflect.getMetadata(metadataKey, descriptor?.value ?? target) ?? [];
accumulatedVal.push(metadataValue);
// .end: tweak
if (descriptor) {
Reflect.defineMetadata(metadataKey, accumulatedVal, descriptor.value);
return descriptor;
}
Reflect.defineMetadata(metadataKey, accumulatedVal, target);
return target;
};
decoratorFactory.KEY = metadataKey;
return decoratorFactory;
}
+13
View File
@@ -0,0 +1,13 @@
import { applyDecorators } from '@nestjs/common';
import { ApiExcludeController, ApiExcludeEndpoint } from '@nestjs/swagger';
export function ApiInternalOnlyEndpoint() {
if (process.env.INTERNAL_SWAGGER !== 'true')
return applyDecorators(ApiExcludeEndpoint());
return () => null;
}
export function ApiInternalOnlyController() {
if (process.env.INTERNAL_SWAGGER !== 'true')
return applyDecorators(ApiExcludeController());
return () => null;
}
@@ -1,17 +1,14 @@
import request from 'supertest';
import { Test } from '@nestjs/testing';
import { HttpStatus, INestApplication } from '@nestjs/common';
import { APP_GUARD } from '@nestjs/core';
import jwt from 'jsonwebtoken';
import { Controller, Get } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { AuthenticationGuard } from './authentication.guard';
import { Controller, Get, HttpStatus, INestApplication } from '@nestjs/common';
import { APP_GUARD } from '@nestjs/core';
import { Test } from '@nestjs/testing';
import jwt from 'jsonwebtoken';
import request from 'supertest';
import { AuthenticationGuard } from '../authentication/authentication.guard';
import { PERMISSIONS_GROUPS } from '../authentication/permissions.enum';
import { AuthClientService } from '../modules/auth/auth.service';
import ErrorCodes from '../utils/errorCodes';
import { User } from './user.decorator';
import { PERMISSIONS_GROUPS } from './permissions.enum';
const logger = {
info: (...args) => args,
@@ -1,4 +1,4 @@
import { createParamDecorator, ExecutionContext } from '@nestjs/common';
import { createParamDecorator, type ExecutionContext } from '@nestjs/common';
import ErrorBuilder from '../utils/ErrorBuilder';
import ErrorCodes from '../utils/errorCodes';
+21 -10
View File
@@ -24,11 +24,10 @@ async function bootstrap() {
optionsSuccessStatus: 204,
},
});
app.use(helmet());
configureSwagger(app);
await app.listen(3333);
if (process.env.KILL_AFTER_START) await app.close();
}
function configureSwagger(app: INestApplication) {
@@ -39,24 +38,36 @@ function configureSwagger(app: INestApplication) {
.replace('\n', '')
.trim();
} catch (error) {
console.log("Can't get branch name", error);
console.warn("Can't get branch name", error);
}
const swaggerTitle = branchName ? `Maestro - ${branchName}` : 'Maestro';
const config = new DocumentBuilder()
const swaggerConfig = new DocumentBuilder()
.setTitle(swaggerTitle)
.setDescription('Documentation for Maestro gateway')
.addBearerAuth(
{ type: 'http', scheme: 'bearer', bearerFormat: 'JWT' },
'Authorization',
)
.build();
.addSecurity('access-token', {
type: 'apiKey',
name: 'Authorization',
in: 'header',
})
.setDescription('This is the Maestro API');
if (process.env.INTERNAL_SWAGGER !== 'true')
swaggerConfig.addServer('https://maestro.dadosfera.ai', 'Dadosfera API');
const config = swaggerConfig.build();
const document = SwaggerModule.createDocument(app, config);
const swaggerJsonFile =
process.env.INTERNAL_SWAGGER === 'true'
? 'docsfera.json'
: 'docsfera.external.json';
if (process.env.ENV === 'local') SwaggerModule.setup('api', app, document);
writeFileSync(
'docsfera.json',
swaggerJsonFile,
JSON.stringify(
{
service: '${DOCS_API_TOKEN}',
+48 -23
View File
@@ -13,7 +13,12 @@ import {
Req,
Param,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import {
ApiHeaders,
ApiOkResponse,
ApiSecurity,
ApiTags,
} from '@nestjs/swagger';
import {
AuthChangePasswordRequest,
AuthResetPasswordRequest,
@@ -28,13 +33,14 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
} from 'src/authentication/authentication.decorator';
} from 'src/decorators/authentication.decorator';
import { AuthClientService } from './auth.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
AuthRefreshAccessTokenReq,
AuthRefreshAccessTokenRes,
AuthSignInReq,
AuthSignInRes,
} from './dtos/login';
@@ -43,8 +49,12 @@ import { AuthGuard } from '@nestjs/passport';
import { Request } from 'express';
import ErrorCodes, { OauthErrors } from 'src/utils/errorCodes';
import jwt from 'jsonwebtoken';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('Auth')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('auth')
export class AuthController {
@@ -74,13 +84,32 @@ export class AuthController {
@HttpCode(HttpStatus.OK)
async signIn(
@Body() { username, password, totp }: AuthSignInReq,
@Headers('Dadosfera-Lang') language: string,
@Language() language: LanguageEnum,
): Promise<AuthSignInRes> {
this.logger.info('/auth - SignIn');
const metadata = PackTheMetadata({ language: language || 'pt-br' });
const metadata = PackTheMetadata({ language });
return this.authClient.signIn({ username, password, totp }, metadata);
}
@Post('refresh-access-token')
@HttpCode(HttpStatus.OK)
@ApiOkResponse({ type: AuthRefreshAccessTokenRes })
async refreshAccessToken(
@Body() body: AuthRefreshAccessTokenReq,
@Language() language: LanguageEnum,
) {
this.logger.info('/auth - RefreshAccessToken');
const { refreshToken, customerName: customer_name } = body;
const metadata = PackTheMetadata({
customer_name,
language,
});
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
}
@ApiInternalOnlyEndpoint()
@Post('/sso/snowflake')
@RequireAllPermissions(PERMISSIONS_GROUPS.SNOWFLAKE.permissions.OPEN)
@HttpCode(HttpStatus.OK)
@@ -96,23 +125,7 @@ export class AuthController {
});
}
@Post('refresh-access-token')
@HttpCode(HttpStatus.OK)
async refreshAccessToken(
@Body()
{ refreshToken, customerName: customer_name }: AuthRefreshAccessTokenReq,
@Headers('Dadosfera-Lang') language: string,
) {
this.logger.info('/auth - RefreshAccessToken');
const metadata = PackTheMetadata({
customer_name,
language,
});
return this.authClient.refreshAccessToken({ refreshToken }, metadata);
}
@ApiInternalOnlyEndpoint()
@Post('change-password')
@HttpCode(HttpStatus.OK)
async changePassword(
@@ -131,11 +144,12 @@ export class AuthController {
});
}
@ApiInternalOnlyEndpoint()
@Post('reset-password')
@HttpCode(HttpStatus.OK)
async resetPassword(
@Body() body: AuthResetPasswordRequest,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('/auth - reset-password');
const metadata = PackTheMetadata({
@@ -147,6 +161,7 @@ export class AuthController {
return this.authClient.resetPassword({ username }, metadata);
}
@ApiInternalOnlyEndpoint()
@Post('verify-reset-password-code')
@HttpCode(HttpStatus.OK)
async verifyResetPasswordCode(
@@ -159,6 +174,7 @@ export class AuthController {
return this.authClient.verifyResetPasswordCode({ username, code });
}
@ApiInternalOnlyEndpoint()
@Post('confirm-reset-password')
@HttpCode(HttpStatus.OK)
async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) {
@@ -173,6 +189,7 @@ export class AuthController {
});
}
@ApiInternalOnlyEndpoint()
@Post('enable-totp')
@HttpCode(HttpStatus.OK)
async enableTotpMFA(
@@ -187,6 +204,7 @@ export class AuthController {
return this.authClient.enableTotpMFA({ accessToken, password });
}
@ApiInternalOnlyEndpoint()
@Post('disable-totp')
@HttpCode(HttpStatus.OK)
async disableTotpMFA(
@@ -201,6 +219,7 @@ export class AuthController {
return this.authClient.disableTotpMFA({ accessToken, password });
}
@ApiInternalOnlyEndpoint()
@Post('dismiss-totp')
@HttpCode(HttpStatus.OK)
async dismissTotpMFA(@Headers() headers) {
@@ -211,6 +230,7 @@ export class AuthController {
return this.authClient.dismissTotpMFA({ accessToken });
}
@ApiInternalOnlyEndpoint()
@Post('verify-totp')
@HttpCode(HttpStatus.OK)
async verifyTotp(@Body() body: AuthVerifyTotpMfaRequest, @Headers() headers) {
@@ -222,17 +242,21 @@ export class AuthController {
return this.authClient.verifyTotp({ accessToken, totp });
}
@ApiInternalOnlyEndpoint()
@Authenticated()
@ApiSecurity('access-token')
@Get('verify-access-token')
verifyAccessToken() {
return { access_token_status: 'valid' };
}
@ApiInternalOnlyEndpoint()
@Get('session/:session')
getSession(@Param('session') session: string) {
return this.authClient.getSession(session);
}
@ApiInternalOnlyEndpoint()
@Get('oauth/google')
@UseGuards(AuthGuard('google-login'))
googleOauth() {
@@ -240,6 +264,7 @@ export class AuthController {
return true;
}
@ApiInternalOnlyEndpoint()
@Get('oauth/google/callback')
@UseGuards(AuthGuard('google-login'))
@Redirect()
+13 -1
View File
@@ -1,3 +1,4 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import {
AuthSignInRequest,
AuthSignInResponse,
@@ -74,6 +75,8 @@ export class AuthCustomer {
tier: string;
@ApiProperty()
scheduleLimit: string;
@ApiProperty()
links: Link[];
}
export class AuthSignInReq implements AuthSignInRequest {
@@ -81,7 +84,10 @@ export class AuthSignInReq implements AuthSignInRequest {
username: string;
@ApiProperty()
password: string;
@ApiPropertyOptional()
@ApiPropertyOptional({
description:
'TOTP code used to login when Multi-Factor Authentication is enabled',
})
totp: string;
}
@@ -106,3 +112,9 @@ export class AuthRefreshAccessTokenReq {
@ApiProperty()
customerName: string;
}
export class AuthRefreshAccessTokenRes {
@ApiProperty()
permissions: string[];
@ApiProperty()
accessToken: string;
}
+83 -17
View File
@@ -6,7 +6,6 @@ import {
ForbiddenException,
Get,
Headers,
HttpException,
Inject,
NotFoundException,
Param,
@@ -15,17 +14,25 @@ import {
Query,
UseFilters,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import {
ApiCreatedResponse,
ApiHeaders,
ApiOkResponse,
ApiTags,
} from '@nestjs/swagger';
import {
Authenticated,
RequireAllPermissions,
RequireSomePermission,
} from '../../authentication/authentication.decorator';
} from '../../decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { CatalogService } from './catalog.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
GetDatasetCatalogTaskRes,
ICatalogAllRequest,
ICatalogAllResponse,
IColumnsMetadataResponse,
ICreateDataAsset,
@@ -35,10 +42,16 @@ import {
IOneDataAsset,
IPreviewResponse,
IUpdateDataRequest,
TriggerCatalogReq,
TriggerCatalogRes,
} from './dtos';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('Catalog')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Controller('catalog')
@UseFilters(new GrpcToHttpExceptionFilter())
@Authenticated()
@@ -59,7 +72,7 @@ export class CatalogController {
)
async searchCatalog(
@User() user: RequestUser,
@Query() query,
@Query() query: ICatalogAllRequest,
): Promise<ICatalogAllResponse> {
const { user_id, customer_name, customer_id, username, permissions } = user;
this.logger.info(`/catalog - searchCatalog`, {
@@ -91,6 +104,7 @@ export class CatalogController {
return res;
}
@ApiInternalOnlyEndpoint()
@Get('data-asset')
async findByPipelineAndObject(@User() user: RequestUser, @Query() query) {
const { username, user_id, customer_id, customer_name, permissions } = user;
@@ -144,6 +158,7 @@ export class CatalogController {
);
}
@ApiInternalOnlyEndpoint()
@Get('tags')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
@@ -174,7 +189,7 @@ export class CatalogController {
)
async getDataAsset(
@User() user: RequestUser,
@Param('id') id,
@Param('id') id: string,
@Query('shared') shared?: 'true',
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
@@ -190,7 +205,7 @@ export class CatalogController {
let has_permission = false;
const metadata = PackTheMetadata({
username,
user_id: undefined,
user_id,
customer_id,
customer_name,
});
@@ -222,6 +237,7 @@ export class CatalogController {
return { data_asset };
}
@ApiInternalOnlyEndpoint()
@Get('data-asset/rls/:id')
@RequireSomePermission(
PERMISSIONS_GROUPS.CATALOG.permissions.GET,
@@ -230,7 +246,7 @@ export class CatalogController {
async getDataAssetRls(
@User() user: RequestUser,
@Headers() headers,
@Param('id') id,
@Param('id') id: string,
) {
const { username, user_id, customer_id, customer_name, permissions } = user;
@@ -284,8 +300,8 @@ export class CatalogController {
)
async getDataAssetColumnsMetadata(
@User() user: RequestUser,
@Headers('Dadosfera-Lang') language,
@Param('id') id,
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IColumnsMetadataResponse> {
const { customer_name, customer_id, user_id, username } = user;
@@ -315,8 +331,8 @@ export class CatalogController {
)
async getDataAssetPreview(
@User() user: RequestUser,
@Headers('Dadosfera-lang') language,
@Param('id') id,
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IPreviewResponse> {
const { customer_name, customer_id, user_id, username } = user;
@@ -345,8 +361,8 @@ export class CatalogController {
)
async getDataAssetDocs(
@User() user: RequestUser,
@Headers('Dadosfera-Lang') language,
@Param('id') id,
@Language() language: LanguageEnum,
@Param('id') id: string,
): Promise<IDocsResponse> {
const { customer_name, customer_id, user_id, username } = user;
@@ -375,8 +391,8 @@ export class CatalogController {
)
async updateDataAsset(
@User() user: RequestUser,
@Headers('Dadosfera-Lang') language,
@Param('id') data_asset_id,
@Language() language: LanguageEnum,
@Param('id') data_asset_id: string,
@Body() body: IUpdateDataRequest,
): Promise<IOneDataAsset> {
const { customer_id, customer_name, user_id, username } = user;
@@ -430,6 +446,7 @@ export class CatalogController {
return res;
}
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/manage-permissions')
async manageDataAssetPermissions(
@Param('id') id: string,
@@ -451,6 +468,7 @@ export class CatalogController {
return { data_asset: JSON.parse(response.data_asset) };
}
@ApiInternalOnlyEndpoint()
@Put('data-asset/:id/revoke-permissions')
async revokeDataAssetPermissions(
@Param('id') id: string,
@@ -561,7 +579,55 @@ export class CatalogController {
username,
});
const response = await this.catalogService.deleteComment(
{ data_asset_id: id, comment: body.comment },
{
data_asset_id: id,
comment: { ...body.comment, created_at: undefined, message: undefined },
},
metadata,
);
return response;
}
@Post('dataset-catalog-task')
@RequireAllPermissions(
PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
)
@ApiCreatedResponse({ type: TriggerCatalogRes })
async triggerCatalog(
@User() user: RequestUser,
@Body() body: TriggerCatalogReq,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const session = await this.catalogService.triggerCatalog(body, metadata);
return { session };
}
@Get('dataset-catalog-task/:session')
@RequireAllPermissions(
PERMISSIONS_GROUPS.CATALOG.permissions.TRIGGER_CATALOG_TASK,
)
@ApiOkResponse({ type: GetDatasetCatalogTaskRes })
async getCatalogTask(
@User() user: RequestUser,
@Param('session') session: string,
) {
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
customer_name,
user_id,
username,
});
const response = await this.catalogService.getDatasetCatalogTask(
session,
metadata,
);
+13 -1
View File
@@ -18,7 +18,7 @@ import { CatalogClientConfiguration } from './catalog-client';
import { UsersService } from '../users/users.service';
import { RolesService } from '../roles/roles.service';
import { Metadata } from '@grpc/grpc-js';
import { IUpdateDataRequest } from './dtos';
import { IUpdateDataRequest, TriggerCatalogReq } from './dtos';
class CatalogService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -333,6 +333,18 @@ class CatalogService implements OnModuleInit {
} as typeof data_asset;
});
}
async triggerCatalog(data: TriggerCatalogReq, metadata: Metadata) {
const { session } = await lastValueFrom(
this.catalogWriteService.TriggerDatasetCataloging(data, metadata),
);
return session;
}
async getDatasetCatalogTask(session: string, metadata: Metadata) {
const res = await lastValueFrom(
this.catalogReadService.GetDatasetCatalogTask({ session }, metadata),
);
return res;
}
}
export { CatalogService };
+84 -3
View File
@@ -1,4 +1,4 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import { ApiProperty, ApiPropertyOptional, PickType } from '@nestjs/swagger';
import { CreateDataAssetRequest } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages';
export enum DataAssetShareType {
@@ -6,6 +6,10 @@ export enum DataAssetShareType {
public = 'public',
private = 'private',
}
export enum OrderEnum {
asc = 'asc',
desc = 'desc',
}
export class EmbedObject {
@ApiProperty()
url: string;
@@ -92,6 +96,8 @@ export class IDataAsset {
comments: IComment[];
@ApiPropertyOptional()
embed?: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type?: DataAssetShareType;
}
export class IOneDataAsset {
@@ -104,9 +110,43 @@ export class IMakeAComment {
message: string;
}
export class CommentToDelete extends PickType(IComment, ['id', 'username']) {}
export class IDeleteComment {
@ApiProperty()
comment: IComment;
comment: CommentToDelete;
}
export class ICatalogAllRequest {
@ApiPropertyOptional({
description: 'Texto usado para filtrar os ativos no catálogo.',
})
search;
@ApiPropertyOptional({
description: 'Número de registros a ser retornado',
maximum: 10000,
minimum: 1,
})
size: number;
@ApiPropertyOptional({
description: 'Usado para paginação, em conjunto com `size`',
minimum: 1,
})
page: number;
@ApiPropertyOptional({
description:
'Campo do data asset para usar na ordenação. Padrão: `created_at` ',
})
sort_by: string;
@ApiPropertyOptional({
enum: OrderEnum,
default: OrderEnum.asc,
description: 'Tipo de ordenação - `asc`: crescente; `desc`: decrescente ',
})
order?: OrderEnum;
}
export class ICatalogAllResponse {
@@ -141,7 +181,7 @@ export class IUpdateDataRequest {
@ApiPropertyOptional()
embed: EmbedObject;
@ApiPropertyOptional({ enum: DataAssetShareType })
share_type: DataAssetShareType;
share_type?: DataAssetShareType;
}
export class ICreateDataAsset implements CreateDataAssetRequest {
@ApiProperty()
@@ -231,3 +271,44 @@ export class IColumnsMetadataResponse {
@ApiProperty({ type: [IColumnMetadata] })
columns_metadata: IColumnMetadata[];
}
export class TriggerCatalogReq {
@ApiProperty({
description: 'Name of the table on Snowflake',
example: 'TB__4DXSD4_TEST',
})
table_name: string;
@ApiPropertyOptional({
description:
'Schema where the asset is located. If not set defaults to "PUBLIC"',
})
table_schema?: string;
@ApiPropertyOptional({
description: 'Id of the pipeline that generated the asset.',
})
pipeline_id: string;
}
export class TriggerCatalogRes {
@ApiProperty()
session: string;
}
export class GetDatasetCatalogTaskRes {
@ApiProperty()
created_at: string;
@ApiProperty()
customer_name: string;
@ApiProperty()
session_id: string;
@ApiProperty()
status: string;
@ApiProperty()
table_name: string;
@ApiProperty()
updated_at: string;
@ApiProperty()
updated_by: string;
}
@@ -9,7 +9,7 @@ import {
ValidationPipe,
} from '@nestjs/common';
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { User, RequestUser } from 'src/authentication/user.decorator';
import { User, RequestUser } from 'src/decorators/user.decorator';
import { ConnectionTestService } from './connection-test.service';
import {
ConnectionTestPingRes,
@@ -24,9 +24,11 @@ import {
GetTableMetadataReq,
} from './dto/connection-test';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Authenticated } from 'src/authentication/authentication.decorator';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('Connection Test')
@Controller('connection-test')
@UseFilters(new GrpcToHttpExceptionFilter())
@@ -19,7 +19,7 @@ import {
CreateConnectionDto,
DatabaseConnectionPropertiesDto,
} from '../connection/dtos/connection';
import { RequestUser } from 'src/authentication/user.decorator';
import { RequestUser } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@Injectable()
+12 -12
View File
@@ -7,19 +7,18 @@ import {
Param,
Delete,
Get,
Headers,
Query,
UseFilters,
} from '@nestjs/common';
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger';
import { ApiExcludeEndpoint, ApiTags } from '@nestjs/swagger';
import { ConnectionClientService } from './client.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Authenticated,
RequireAllPermissions,
} from 'src/authentication/authentication.decorator';
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { ValidationPipe } from '../../pipes/object-validation.pipe';
import {
ConnectionDetailsRes,
@@ -30,11 +29,13 @@ import {
import { CreateConnectionDto } from './dtos/connection';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
const connectionPermissions = PERMISSIONS_GROUPS.CONNECTION.permissions;
@UseFilters(new GrpcToHttpExceptionFilter())
@ApiTags('connections')
@ApiBearerAuth()
@Authenticated()
@Controller('connections')
export class ConnectionController {
@@ -47,7 +48,9 @@ export class ConnectionController {
this.logger = dadosferaLogger.logger;
}
@ApiInternalOnlyEndpoint()
@Get('connections-migration')
@ApiExcludeEndpoint()
@RequireAllPermissions(
PERMISSIONS_GROUPS.DADOSFERA.permissions.CONNECTIONS_MIGRATION,
)
@@ -83,12 +86,11 @@ export class ConnectionController {
@RequireAllPermissions(connectionPermissions.CREATE)
async updateConnection(
@User() user: RequestUser,
@Headers('Dadosfera-Lang') language: string,
@Language() language: LanguageEnum,
@Param('id') id: string,
@Body(new ValidationPipe()) body: UpdateConnectionDto,
): Promise<ConnectionRes> {
this.logger.info('/connections - Update Connection');
if (!language) language = 'en-us';
const { user_id, customer_id, customer_name, username } = user;
const metadata = PackTheMetadata({
@@ -135,11 +137,10 @@ export class ConnectionController {
@Get()
async getAllConnections(
@User() user: RequestUser,
@Headers('Dadosfera-Lang') language,
@Language() language: LanguageEnum,
@Query() queries,
): Promise<ConnectionsRes> {
this.logger.info('/connections - Get All Connection');
if (!language) language = 'en-us';
const { user_id, customer_id, customer_name, username } = user;
const { search, size, page, ...filters } = queries;
@@ -161,13 +162,12 @@ export class ConnectionController {
@Get('/:id')
async getConnectionDetails(
@Headers('Dadosfera-Lang') language,
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Param('id') id,
@Param('id') id: string,
@Query() queries,
): Promise<ConnectionDetailsRes> {
this.logger.info('/connections - Get Connection Details');
if (!language) language = 'en-us';
console.log(queries);
+12 -17
View File
@@ -11,11 +11,10 @@ import {
UploadedFile,
UseInterceptors,
Inject,
Headers,
UploadedFiles,
} from '@nestjs/common';
import { FileInterceptor, FilesInterceptor } from '@nestjs/platform-express';
import { ApiBearerAuth, ApiConsumes, ApiTags } from '@nestjs/swagger';
import { ApiConsumes, ApiTags } from '@nestjs/swagger';
import { ConnectorClientService } from './client.service';
import { AddTagDto } from './dtos/add-tag';
import { CreateConnectorDto } from './dtos/create-connector';
@@ -27,11 +26,14 @@ import {
Authenticated,
RequireAllPermissions,
RequireSomePermission,
} from 'src/authentication/authentication.decorator';
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('connectors')
@ApiBearerAuth()
@Authenticated()
@Controller('connectors')
export class ConnectorController {
@@ -98,11 +100,10 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
async getAllConnectors(
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
@Query() queries: GetAllDto,
) {
this.logger.info('/GET - getAllConnectors Route');
if (!language) language = 'en-us';
const { search, size, page, ...filters } = queries;
@@ -143,12 +144,11 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
async getConnector(
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
@Param('plugin') plugin: string,
@Query('version') version: string,
) {
this.logger.info('/GET/:plugin - getConnector Route');
if (!language) language = 'en-us';
const pluginId = `${plugin}-${version}`;
@@ -172,13 +172,11 @@ export class ConnectorController {
PERMISSIONS_GROUPS.PIPELINE.permissions.DELETE,
)
async getConnectorDetails(
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
@Param('plugin') plugin: string,
@Query('version') version: string,
) {
this.logger.info('/upload - Upload Connector Route');
if (!language) language = 'en-us';
const pluginId = `${plugin}-${version}`;
const response: any = await this.connectorClientService.getConnectorDetails(
@@ -275,16 +273,13 @@ export class ConnectorController {
@Param('plugin') plugin: string,
@Query('version') version: string,
) {
this.logger.info('/upload - Upload Connector Route');
this.logger.info('DELETE /:plugin - Delete Connector Route');
const response: any = await this.connectorClientService.deleteConnector(
const response = await this.connectorClientService.deleteConnector(
plugin,
version,
);
return {
message: response.message || 'ok',
connector: { ...JSON.parse(response.connector) },
};
return { message: response.message, connector: response.connector };
}
}
@@ -0,0 +1,56 @@
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { IdResponse } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import {
Body,
Controller,
Get,
Inject,
Param,
Put,
UseFilters,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
} from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { CustomersService } from './customers.service';
import { CustomerDto, CustomerLinksResponse } from './dtos/customers';
@ApiInternalOnlyController()
@ApiTags('Customers')
@Controller('customers')
@Authenticated()
@UseFilters(GrpcToHttpExceptionFilter)
export class CustomersController {
logger: DadosferaLogger;
constructor(
@Inject(DadosferaLogger)
dadosferaLogger: DadosferaLogger,
private customersService: CustomersService,
) {
this.logger = dadosferaLogger.logger;
}
@Get(':id/links')
async getCustomerLinks(@Param('id') id): Promise<CustomerLinksResponse> {
this.logger.info('getCustomerLinks', { id });
const links = await this.customersService.getLinks(id);
return { links };
}
@Put(':id/links')
@RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN)
setCustomerLinks(
@Body() body: CustomerDto,
@Param('id') id,
): Promise<IdResponse> {
const { links } = body;
this.logger.info('setCustomerLinks', { id, links });
return this.customersService.setLinks(id, links);
}
}
+16
View File
@@ -0,0 +1,16 @@
import { Module } from '@nestjs/common';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ClientsModule } from '@nestjs/microservices';
import { DucClient } from '../duc/client.config';
import { CustomersController } from './customers.controller';
import { CustomersService } from './customers.service';
const client = new DucClient();
@Module({
imports: [ClientsModule.register([client.providerOptions])],
controllers: [CustomersController],
providers: [CustomersService, DadosferaLogger],
exports: [CustomersService],
})
export class CustomersModule {}
@@ -0,0 +1,63 @@
import {
OnModuleInit,
Inject,
Injectable,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { firstValueFrom, lastValueFrom } from 'rxjs';
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { DucClient } from '../duc/client.config';
import { ClientGrpc } from '@nestjs/microservices';
import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc';
import { CustomerUpdateRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service';
import ErrorCodes from 'src/utils/errorCodes';
@Injectable()
export class CustomersService implements OnModuleInit {
private customerService: CustomersProtoService;
constructor(
@Inject(DucClient.name) private readonly grpcClient: ClientGrpc,
) {}
onModuleInit() {
this.customerService = this.grpcClient.getService<CustomersProtoService>(
ProtoServices.CustomersProtoService,
);
}
async getLinks(customerId: string) {
try {
const result = await lastValueFrom(
this.customerService.CustomerFindOneById({ id: customerId }),
);
return result.customer?.links || [];
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
throw err;
}
}
async setLinks(customerId: string, links: Link[]) {
if (!customerId || !links) {
throw new HttpException(null, HttpStatus.BAD_REQUEST);
}
try {
return await firstValueFrom(
this.customerService.CustomerUpdate({
id: customerId,
links,
} as CustomerUpdateRequest),
);
} catch (err) {
if (err.details === ErrorCodes.CUSTOMER.NOT_FOUND)
throw new HttpException(err.details, HttpStatus.NOT_FOUND);
else throw err;
}
}
}
+12
View File
@@ -0,0 +1,12 @@
import { Link } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/entities';
import { ApiProperty } from '@nestjs/swagger';
export class CustomerDto {
@ApiProperty()
links: Link[];
}
export class CustomerLinksResponse {
@ApiProperty()
links: Link[];
}
+8 -4
View File
@@ -12,7 +12,7 @@ import {
import { InputsService } from './inputs.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { AuthenticateCondition } from 'src/authentication/authentication.decorator';
import { AuthenticateCondition } from 'src/decorators/authentication.decorator';
import { ApiOkResponse, ApiTags } from '@nestjs/swagger';
import {
CreateInputReq,
@@ -21,9 +21,10 @@ import {
Input,
} from './dtos/input.model';
import { UpdateInputRequest } from './dtos/old_interfaces';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { Info } from '@dadosfera/protospack-v2/dist/lib/Input/interfaces/entities';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('Inputs')
@Controller('inputs')
@@ -66,6 +67,7 @@ export class InputsController {
this.logger = dadosferaLogger.logger;
}
@ApiInternalOnlyEndpoint()
@Get('available-entities/:plugin')
@ApiOkResponse({ type: GetAvailableEntitiesRes })
async getAvailableEntities(
@@ -102,6 +104,7 @@ export class InputsController {
return response;
}
@ApiInternalOnlyEndpoint()
@Get()
async findAll(@User() user: RequestUser) {
const { user_id, customer_id, customer_name: customer } = user;
@@ -120,8 +123,7 @@ export class InputsController {
}
@Get('/:id')
async findOne(@Body() body, @Param() params) {
const { id } = params;
async findOne(@Body() body, @Param('id') id: string) {
this.logger.info(`/input/${id} - ON FIND ONE ROUTE`, {
user: body.info.user_id,
customer: body.info.customer,
@@ -132,6 +134,7 @@ export class InputsController {
return response;
}
@ApiInternalOnlyEndpoint()
@Patch(':id')
async update(@Body() updateInputDto: UpdateInputRequest, @Param() params) {
const { id } = params;
@@ -147,6 +150,7 @@ export class InputsController {
return response;
}
@ApiInternalOnlyEndpoint()
@Delete(':id')
async delete(@Body() data, @Param() params) {
const { id } = params;
+4 -2
View File
@@ -1,8 +1,10 @@
import { Body, Controller, Inject, Param, Post } from '@nestjs/common';
import { init } from 'mixpanel';
import { Authenticated } from 'src/authentication/authentication.decorator';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
@ApiInternalOnlyController()
@Authenticated()
@Controller('trackEvent')
export class MixpanelController {
@@ -6,6 +6,8 @@ import {
} from '@nestjs/microservices';
import { NetworkConfig } from '@dadosfera/protospack-v2';
const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
export class NetworkConfigGrpcClient {
public name = 'NetworkConfigGrpcClient';
@@ -17,8 +19,7 @@ export class NetworkConfigGrpcClient {
NetworkConfig.ProtoPackages.ReadPackage,
NetworkConfig.ProtoPackages.WritePackage,
],
credentials:
process.env.ENV === 'local' ? undefined : credentials.createSsl(),
credentials: isLocalConnection ? undefined : credentials.createSsl(),
protoPath: [
NetworkConfig.ProtoPaths.ReadFilePath,
NetworkConfig.ProtoPaths.WriteFilePath,
@@ -1,12 +1,9 @@
import {
Body,
Controller,
Delete,
Get,
Headers,
Inject,
Param,
Patch,
Post,
Query,
UploadedFile,
@@ -15,7 +12,7 @@ import {
import { NetworkConfigService } from './network-config.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ApiConsumes, ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import {
CreateNetworkConfigReq,
CreateNetworkConfigRes,
@@ -26,7 +23,9 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
} from 'src/authentication/authentication.decorator';
} from 'src/decorators/authentication.decorator';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
const networkConfigPermissions = PERMISSIONS_GROUPS.NETWORK_CONFIG.permissions;
@ApiTags('Network Config')
@@ -40,18 +39,18 @@ export class NetworkConfigController {
) {
this.logger = dadosferaLogger.logger;
}
@Get()
@ApiOkResponse({ type: FindAllNetworkConfigsRes })
async findAllNetworkConfigs(
@User() user: RequestUser,
@Headers('Dadosfera-Lang') language,
@Language() language: LanguageEnum,
@Query() queries,
) {
this.logger.info('GET /network-config', {
user: user.user_id,
customer: user.customer_name,
});
if (!language) language = 'en-us';
const { user_id, customer_id, customer_name } = user;
const { search, size, page, ...filters } = queries;
@@ -79,14 +78,14 @@ export class NetworkConfigController {
@ApiOkResponse({ type: FindAllNetworkConfigsRes })
async findNetworkConfigByIdDetails(
@User() user: RequestUser,
@Headers('Dadosfera-Lang') language,
@Language() language: LanguageEnum,
@Param('id') id: string,
) {
this.logger.info('GET /network-config/:id', {
user: user.user_id,
customer: user.customer_name,
});
if (!language) language = 'en-us';
return this.networkConfigService.findNetworkConfigByIdDetails(id, {
customer_name: user.customer_name,
customer_id: user.customer_id,
+7 -1
View File
@@ -5,6 +5,7 @@ import { ConnectionClientService } from '../connection/client.service';
import jwt from 'jsonwebtoken';
import DadosferaLogger from '@dadosfera/dadosfera-logs/dist';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@ApiTags('oauth')
@Controller('oauth')
export class OauthController {
@@ -15,7 +16,6 @@ export class OauthController {
private logger: DadosferaLogger,
) {
switch (process.env.ENV) {
case 'dev':
case 'stg':
this.redirectUrl = `https://app.${process.env.ENV}.dadosfera.ai/collect/auth-callback`;
break;
@@ -32,6 +32,7 @@ export class OauthController {
return true;
}
@ApiInternalOnlyEndpoint()
@Get('hubspot/callback')
@UseGuards(AuthGuard('hubspot'))
@Redirect()
@@ -45,6 +46,7 @@ export class OauthController {
return true;
}
@ApiInternalOnlyEndpoint()
@Get('googleads/callback')
@UseGuards(AuthGuard('google'))
@Redirect()
@@ -58,6 +60,7 @@ export class OauthController {
return true;
}
@ApiInternalOnlyEndpoint()
@Get('google-sheets/callback')
@UseGuards(AuthGuard('google'))
@Redirect()
@@ -71,6 +74,7 @@ export class OauthController {
return true;
}
@ApiInternalOnlyEndpoint()
@Get('facebook/callback')
@UseGuards(AuthGuard('facebook'))
@Redirect()
@@ -84,6 +88,7 @@ export class OauthController {
return true;
}
@ApiInternalOnlyEndpoint()
@Get('mailchimp/callback')
@UseGuards(AuthGuard('mailchimp'))
@Redirect()
@@ -97,6 +102,7 @@ export class OauthController {
return true;
}
@ApiInternalOnlyEndpoint()
@Get('salesforce/callback')
@UseGuards(AuthGuard('salesforce'))
@Redirect()
@@ -29,7 +29,6 @@ export class GoogleStrategy extends PassportStrategy(Strategy, 'google') {
super(options, verify);
switch (process.env.ENV) {
case 'dev':
case 'stg':
this.redirect_uri = `https://maestro.${process.env.ENV}.dadosfera.ai/oauth`;
break;
@@ -17,7 +17,7 @@ export class HubspotStrategy extends PassportStrategy(Strategy) {
clientSecret: oauthSecrets.hubspot.client_secret,
callbackURL: oauthSecrets.hubspot.redirect_uri,
scope:
'automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
'tickets automation business-intelligence oauth forms integration-sync sales-email-read crm.lists.read crm.objects.contacts.read crm.schemas.contacts.read crm.objects.companies.read crm.objects.deals.read crm.schemas.companies.read crm.schemas.deals.read crm.objects.owners.read crm.objects.quotes.read crm.schemas.quotes.read crm.objects.line_items.read crm.schemas.line_items.read',
passReqToCallback: true,
},
(accessToken, refreshToken, tokenInfo, profile, done) => {
@@ -1,13 +1,16 @@
import { Controller, Get, Headers, Inject } from '@nestjs/common';
import { Controller, Get, Inject } from '@nestjs/common';
import { ApiHeader, ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Authenticated } from 'src/authentication/authentication.decorator';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { PermissionUsages } from 'src/authentication/permissions.enum';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { GetPublicPermissionsRes } from './dto/entities';
import { PermissionsService } from './permissions.service';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('Permissions')
@Authenticated()
@Controller('permissions')
@@ -32,10 +35,10 @@ export class PermissionsController {
@ApiOkResponse({ type: GetPublicPermissionsRes })
getPublicPermissions(
@User() user: RequestUser,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('getPublicPermissions', { user });
if (!language) language = 'en-us';
const permissionGroups = this.permissionsService.getPermissionsGrouped(
PermissionUsages.PUBLIC,
language,
@@ -60,10 +60,10 @@ export class PermissionsService
}).catch((err: ErrorBuilder) => {
if (
err.code === 'No connection established' &&
(process.env.LOCAL_ENV === 'true' || process.env.ENV === 'local')
process.env.ENV === 'local'
) {
return this.logger.info(
"couldn't connect to DUC. suppresing in local env",
"couldn't connect to DUC. suppressing in local env",
);
}
+16 -6
View File
@@ -1,13 +1,15 @@
import { Body, Controller, Get, Inject, Param, Post } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { ApiOperation, ApiTags } from '@nestjs/swagger';
import {
AuthenticateCondition,
Authenticated,
} from 'src/authentication/authentication.decorator';
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('Pipelines')
@Controller('pipelines')
@Authenticated()
@@ -42,8 +44,12 @@ export class PipelinesController {
}
@Post('start/:id')
async activate(@Param() params, @Body() body) {
const { id } = params;
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/start/:id instead',
})
async activate(@Param('id') id: string, @Body() body) {
const { info } = body;
this.logger.info(
@@ -60,8 +66,12 @@ export class PipelinesController {
}
@Get(':id/status')
async getPipelineStatus(@Body() body, @Param() params) {
const { id } = params;
@ApiOperation({
deprecated: true,
description:
'This method is deprecated. Please use route /pipelinesV2/:id/status instead',
})
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(
+18
View File
@@ -10,6 +10,8 @@ export class IPipelineV2 {
@ApiProperty()
description: string;
@ApiProperty()
type: string;
@ApiProperty()
connection_id: string;
@ApiProperty()
cron: string;
@@ -104,4 +106,20 @@ export class ICreatePipelineCSVFile {
@ApiProperty()
header: boolean;
@ApiProperty()
engine: string;
}
export class PipelineFindAllReq {
@ApiPropertyOptional()
search?: string | undefined;
@ApiPropertyOptional()
filters?: string | undefined;
@ApiPropertyOptional()
size?: string | undefined;
@ApiPropertyOptional()
page?: string | undefined;
@ApiPropertyOptional()
type?: string | undefined;
}
+96 -54
View File
@@ -7,28 +7,30 @@ import {
Param,
Post,
Put,
Headers,
Query,
UseFilters,
HttpCode,
HttpStatus,
Patch,
HttpException,
BadRequestException,
} from '@nestjs/common';
import {
ApiCreatedResponse,
ApiHeaders,
ApiNoContentResponse,
ApiOperation,
ApiTags,
} from '@nestjs/swagger';
import {
AuthenticateCondition,
RequireAllPermissions,
} from 'src/authentication/authentication.decorator';
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { PipelinesService } from './pipelines.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { Messages } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import { PipelinesService as OldPipelineService } from 'src/modules/pipelines/pipelines.service';
@@ -38,11 +40,16 @@ import {
ICreatePipelineV2Req,
IPipelineV2,
IInitUploadCSVFile,
PipelineFindAllReq,
} from './interfaces';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator';
@UseFilters(new GrpcToHttpExceptionFilter())
@ApiTags('PipelinesV2')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('pipelinesV2')
@AuthenticateCondition((req, user) => {
let action;
@@ -82,12 +89,12 @@ export class PipelinesController {
@Post()
@ApiCreatedResponse({ type: IPipelineV2 })
async create(
@Headers('Dadosfera-Lang') language,
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Body() createPipelineDto: ICreatePipelineV2Req,
) {
this.logger.info('PipelinesController - create', { user });
if (!language) language = 'en-us';
const { customer_id, customer_name, username, user_id } = user;
const metadata = PackTheMetadata({
customer_id,
@@ -108,13 +115,11 @@ export class PipelinesController {
@Get()
async findAll(
@User() user: RequestUser,
@Headers('Dadosfera-Lang') language,
@Query() data,
@Language() language: LanguageEnum,
@Query() data: PipelineFindAllReq,
) {
this.logger.info('PipelinesController - findAll', { user });
if (!language) language = 'en-us';
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
customer_id,
@@ -134,16 +139,13 @@ export class PipelinesController {
@Get('/download-logs')
async downloadLogs(
@User() user: RequestUser,
@Headers('Dadosfera-Lang') language,
@Query() query,
@Language() language: LanguageEnum,
@Query('pipeline_run_id') pipeline_run_id: string,
) {
this.logger.info('PipelinesController - downloadLogs', { user });
if (!language) language = 'en-us';
if (!query) {
throw new HttpException('Query params not provided', 400);
}
if (!pipeline_run_id)
throw new BadRequestException('Missing pipeline_run_id');
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
@@ -155,7 +157,7 @@ export class PipelinesController {
});
const url = await this.pipelinesClientService.downloadLogs(
query.pipeline_run_id,
pipeline_run_id,
metadata,
);
@@ -166,42 +168,51 @@ export class PipelinesController {
@Get(':id/config')
async getPipelineproperties(
@Headers('Dadosfera-Lang') language,
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Param('id') id,
@Param('id') id: string,
) {
this.logger.info('PipelinesController - getPipelineproperties', { user });
const metadata = PackTheMetadata({
...user,
language: language || 'pt-br',
});
const metadata = PackTheMetadata({ ...user, language: language });
return this.pipelinesClientService.findOneProperties(id, metadata);
}
@Get(':id/objects')
async getPipelineObjects(
@Headers('Dadosfera-Lang') language,
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Param('id') id,
@Param('id') id: string,
) {
this.logger.info('PipelinesController - getPipelineObjects', { user });
const metadata = PackTheMetadata({
...user,
language: language || 'pt-br',
});
const metadata = PackTheMetadata({ ...user, language: language });
return this.pipelinesClientService.findOneObjects(id, metadata);
}
@Get(':id/status')
async getPipelineStatus(@Body() body, @Param('id') id: string) {
body.id = id;
this.logger.info(
process.env.DEV_URL + `/pipeline/${id} - ON GET PIPELINE STATUS ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.oldPipelinesService.getPipelineStatus(body);
return response;
}
@Get('/:id')
async findOne(
@Headers('Dadosfera-Lang') language,
@Language() language: LanguageEnum,
@User() user: RequestUser,
@Param('id') id,
@Param('id') id: string,
): Promise<Messages.PipelineV2FindOneResponse> {
this.logger.info('PipelinesController - findOne', { user });
if (!language) language = 'en-us';
const { customer_id, customer_name, user_id, username } = user;
const metadata = PackTheMetadata({
@@ -235,11 +246,11 @@ export class PipelinesController {
return result;
}
@Put('/:id')
@Patch('/:id')
async update(
@Headers('Dadosfera-Lang') language,
@Language() language: LanguageEnum,
@Body() updatePipelineDto,
@Param('id') id,
@Param('id') id: string,
@User() user: RequestUser,
) {
this.logger.info('PipelinesController - update', { user });
@@ -247,7 +258,6 @@ export class PipelinesController {
delete updatePipelineDto.info;
const { customer_id, customer_name, user_id, username } = user;
if (!language) language = 'en-us';
const metadata = PackTheMetadata({
customer_id,
@@ -270,25 +280,29 @@ export class PipelinesController {
return response;
}
@Patch('/:id')
async updateByPatch(
@Headers('Dadosfera-Lang') language,
@ApiInternalOnlyEndpoint()
@Put('/:id')
@ApiOperation({
deprecated: true,
description: 'This method is deprecated. Please use PATCH instead',
})
async updateDeprecated(
@Language() language: LanguageEnum,
@Body() updatePipelineDto,
@Param('id') id,
@Param('id') id: string,
@User() user: RequestUser,
) {
this.logger.info('PipelinesController - patch', { user });
this.logger.info('PipelinesController - put', { user });
const response = await this.update(language, updatePipelineDto, id, user);
this.logger.info('PipelinesController - patch: OK', { user });
this.logger.info('PipelinesController - put: OK', { user });
return response;
}
@Delete(':id')
@ApiNoContentResponse()
@HttpCode(HttpStatus.NO_CONTENT)
async delete(@Param() params, @User() user: RequestUser) {
async delete(@Param('id') id: string, @User() user: RequestUser) {
this.logger.info('PipelinesController - delete', { user });
const { id } = params;
const metadata = PackTheMetadata({
customer_id: user.customer_id,
customer_name: user.customer_name,
@@ -298,6 +312,7 @@ export class PipelinesController {
this.logger.info('PipelinesController - delete: OK');
}
@ApiInternalOnlyEndpoint()
@Post('/init-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async initUploadFile(
@@ -329,6 +344,7 @@ export class PipelinesController {
return { urls: JSON.parse(urls), upload_id };
}
@ApiInternalOnlyEndpoint()
@Post('/complete-upload')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async completeUploadFile(
@@ -346,36 +362,44 @@ export class PipelinesController {
);
}
@ApiInternalOnlyEndpoint()
@Post('/file')
@RequireAllPermissions(PERMISSIONS_GROUPS.PIPELINE.permissions.CREATE)
async uploadedFile(
@User() user: RequestUser,
@Body() body: ICreatePipelineCSVFile,
) {
this.logger.info('/upload - Upload Connector Route');
this.logger.info('/file - Upload Connector Route');
const metadata = PackTheMetadata({ ...user });
const { name, sep, header, encoding, description, file_name } = body;
const { name, sep, header, encoding, description, file_name, engine } =
body;
const file_format_params = {
...(sep && { sep }),
...(encoding && { encoding }),
...(header !== undefined ? { header } : {}),
};
const source_prefix = `${user.customer_name}/${file_name}`;
const upload_pipeline = {
const upload_pipeline: ICreatePipelineV2Req = {
connection_id: process.env.UPLOAD_FILE_AGENT_CONNECTION,
connector_name: 'Amazon S3',
connector_plugin: 'aws_s3',
connector_version: '1.0.0',
image_url: 'https://assets.dadosfera.ai/images/connectors/csv.svg',
image_url: `https://assets.dadosfera.ai/images/connectors/${engine}.svg`,
name,
description,
transformations_ids: [],
tags: [],
cron: '@once',
config: { cron: '@once', tables: [] },
type: 'upload',
properties: {
engine: 'csv',
engine,
source_bucket: process.env.BUCKET_CUSTOMER_CSV_ASSETS,
source_prefix,
file_format_params: { sep, encoding, header: Boolean(header) },
is_a_upload_csv: true,
file_format_params,
},
input_id: undefined,
};
@@ -387,4 +411,22 @@ export class PipelinesController {
return pipeline;
}
@ApiInternalOnlyEndpoint()
@Post('start/:id')
async activate(@Param('id') id: string, @Body() body) {
const { info } = body;
this.logger.info(
process.env.DEV_URL + `/pipeline/start/${id} - ON START PIPELINE ROUTE`,
{
user: body.info.user_id,
customer: body.info.customer,
},
);
const response = await this.oldPipelinesService.runPipeline({ id, info });
return response;
}
}
+29 -15
View File
@@ -21,9 +21,9 @@ import { PipelineV2CreateRequest } from '@dadosfera/protospack-v2/dist/lib/Pipel
import { Metadata } from '@grpc/grpc-js';
import { ConnectorClientService } from '../connector/client.service';
import { InputsService } from '../inputs/inputs.service';
import { RequestUser } from 'src/authentication/user.decorator';
import { RequestUser } from 'src/decorators/user.decorator';
import { TransformationsService } from '../transformations/transformations.service';
import { getObjValueFromPath } from 'src/utils/ObjValueFromPath';
import { getObjValueFromPath, objHasPath } from 'src/utils/ObjValueFromPath';
import ErrorCodes from 'src/utils/errorCodes';
import ErrorBuilder from 'src/utils/ErrorBuilder';
@@ -220,7 +220,7 @@ export class PipelinesService implements OnModuleInit {
this.logger.info('PipelinesClientService - findOneProperties');
const { pipeline } = await this.findOne({ id }, metadata);
const pipelineProperties = JSON.parse(pipeline.properties);
const properties = JSON.parse(pipeline.properties);
const connectorId = `${pipeline.connector_plugin}-${pipeline.connector_version}`;
@@ -228,24 +228,38 @@ export class PipelinesService implements OnModuleInit {
connectorId,
metadata.get('language')[0].toString(),
);
const connector = JSON.parse(res.connector);
const config_controls = connector.config_controls || [];
const connector: { [key: string]: any; config_controls: any[] } =
JSON.parse(res.connector);
const config_controls = (connector.config_controls || []).filter(
(config_control) => {
const path: string = config_control.name;
if (!objHasPath({ object: { properties }, path })) return false;
for (const config_control of config_controls) {
const path: string = config_control.name;
const default_value = getObjValueFromPath({
object: { properties: pipelineProperties },
path,
});
if (default_value != undefined)
config_control.default_value = default_value;
}
let default_value = getObjValueFromPath({
object: { properties },
path,
});
if (default_value != undefined) {
if (
path === 'properties.spreadsheet_id' &&
typeof default_value === 'string'
)
default_value = 'https://docs.google.com/spreadsheets/d/'.concat(
default_value,
);
config_control.default_value = default_value;
}
return true;
},
);
return {
pipeline: {
...pipeline,
config_controls,
properties: pipelineProperties,
properties,
},
};
}
@@ -1,12 +1,16 @@
import { Body, Controller, HttpException, Post } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import axios from 'axios';
import { User, RequestUser } from 'src/authentication/user.decorator';
import { Authenticated } from 'src/decorators/authentication.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { User, RequestUser } from 'src/decorators/user.decorator';
import { getSecretFromSecretsManager } from 'src/utils/SecretManager';
import { INote } from './dtos';
@ApiInternalOnlyController()
@ApiTags('Productboard')
@Controller('productboard')
@Authenticated()
export class ProductboardController {
@Post('notes')
async sendNote(@Body() note: INote, @User() user: RequestUser) {
+16 -19
View File
@@ -3,7 +3,6 @@ import {
Controller,
Delete,
Get,
Headers,
Inject,
Param,
Patch,
@@ -14,11 +13,11 @@ import {
} from '@nestjs/common';
import {
ApiCreatedResponse,
ApiHeader,
ApiHeaders,
ApiOkResponse,
ApiTags,
} from '@nestjs/swagger';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RolesService } from './roles.service';
import {
@@ -43,16 +42,14 @@ import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
} from 'src/authentication/authentication.decorator';
} from 'src/decorators/authentication.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('Roles')
@ApiHeader({
name: 'dadosfera-lang',
enum: LanguageEnum,
required: false,
description: 'Defaults to pt-br',
})
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Authenticated()
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('roles')
@@ -72,7 +69,7 @@ export class RolesController {
async searchRoles(
@User() user: RequestUser,
@Query() params: GetRolesByCustomerReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('searchRoles', { user });
this.rolesService.setLanguage(language);
@@ -85,7 +82,7 @@ export class RolesController {
async createRole(
@User() user: RequestUser,
@Body() body: CreateRoleReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('createRole', { user });
this.rolesService.setLanguage(language);
@@ -98,7 +95,7 @@ export class RolesController {
async grantPermissionsToRole(
@User() user: RequestUser,
@Body() body: GrantPermissionsToRoleReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('grantPermissionsToRole', { user });
this.rolesService.setLanguage(language);
@@ -111,7 +108,7 @@ export class RolesController {
async revokePermissionsFromRole(
@User() user: RequestUser,
@Body() body: RevokePermissionsFromRoleReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('revokePermissionsToRole', { user });
this.rolesService.setLanguage(language);
@@ -124,7 +121,7 @@ export class RolesController {
async setRolePermissions(
@User() user: RequestUser,
@Body() body: SetRolePermissionsReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('setRolePermissions', { user });
this.rolesService.setLanguage(language);
@@ -137,7 +134,7 @@ export class RolesController {
async setRoleUsers(
@User() user: RequestUser,
@Body() body: SetRoleUsersReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('setRoleUsers', { user });
this.rolesService.setLanguage(language);
@@ -150,7 +147,7 @@ export class RolesController {
async getRoleById(
@User() user: RequestUser,
@Param('id') id: string,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('getRoleById', { user });
this.rolesService.setLanguage(language);
@@ -162,7 +159,7 @@ export class RolesController {
async deleteRole(
@User() user: RequestUser,
@Param('id') id: string,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('deleteRole', { user });
this.rolesService.setLanguage(language);
@@ -176,7 +173,7 @@ export class RolesController {
@User() user: RequestUser,
@Param('id') id: string,
@Body() body: UpdateRoleReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('updateRole', { user });
this.rolesService.setLanguage(language);
+1 -1
View File
@@ -18,7 +18,7 @@ import { Permission } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/ent
import { lastValueFrom } from 'rxjs';
import { PermissionsService } from '../permissions/permissions.service';
import { LanguageEnum } from 'src/utils/languages.enum';
import { RequestUser } from 'src/authentication/user.decorator';
import { RequestUser } from 'src/decorators/user.decorator';
import { DucClient } from '../duc/client.config';
interface GetRolesPermissionsName {
@@ -9,17 +9,22 @@ import {
Headers,
UseFilters,
} from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { ApiHeaders, ApiTags } from '@nestjs/swagger';
import { TermsOfUseAcceptRequest } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/messages';
import { TermsOfUseClientService } from './termsOfUse.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { GrpcToHttpExceptionFilter } from '../../error/grpc-to-http-exception.filter';
import { RequestUser, User } from '../../authentication/user.decorator';
import { RequestUser, User } from '../../decorators/user.decorator';
import { Metadata } from '@grpc/grpc-js';
import { Language } from 'src/decorators/language.decorator';
import { LanguageEnum } from 'src/utils/languages.enum';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
@ApiInternalOnlyController()
@ApiTags('TermsOfUse')
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@UseFilters(new GrpcToHttpExceptionFilter())
@Controller('terms-of-use')
export class TermsOfUseController {
@@ -36,7 +41,7 @@ export class TermsOfUseController {
@Get('token')
async getToken(
@User({ required: true }) user: RequestUser,
@Headers('Dadosfera-Lang') language: string,
@Language() language: LanguageEnum,
) {
this.logger.info('/terms-of-use - get token');
const metadata = new Metadata();
@@ -12,15 +12,17 @@ import { ApiTags } from '@nestjs/swagger';
import { TransformationsClientService } from './client.service';
import { IIdRequest } from './interfaces';
import {
AuthenticateCondition,
Authenticated,
RequireSomePermission,
} from 'src/authentication/authentication.decorator';
} from 'src/decorators/authentication.decorator';
import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum';
import { TransformationsService } from './transformations.service';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { RequestUser, User } from 'src/authentication/user.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
const pipelinePermissions = PERMISSIONS_GROUPS.PIPELINE.permissions;
@ApiInternalOnlyController()
@ApiTags('Transformations')
@Controller('transformations')
@Authenticated()
@@ -70,8 +72,7 @@ export class TransformationsController {
@Get('/:id')
@RequireSomePermission(pipelinePermissions.GET)
async findOne(@Body() data, @Param() params) {
const { id } = params;
async findOne(@Body() data, @Param('id') id: string) {
this.logger.info(`/transformation/${id} - ON Find One ROUTE`, {
user: data.info.user_id,
customer: data.info.customer,
+26 -29
View File
@@ -1,9 +1,9 @@
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import {
Body,
Controller,
Delete,
Get,
Headers,
HttpCode,
HttpStatus,
Inject,
@@ -16,18 +16,22 @@ import {
} from '@nestjs/common';
import {
ApiCreatedResponse,
ApiHeader,
ApiHeaders,
ApiOkResponse,
ApiTags,
} from '@nestjs/swagger';
import { DadosferaLogger } from '@dadosfera/dadosfera-logs';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import {
Authenticated,
RequireAllPermissions,
} from 'src/authentication/authentication.decorator';
import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum';
import { RequestUser, User } from 'src/authentication/user.decorator';
} from 'src/decorators/authentication.decorator';
import { Language } from 'src/decorators/language.decorator';
import { ApiInternalOnlyController } from 'src/decorators/swagger.decorator';
import { RequestUser, User } from 'src/decorators/user.decorator';
import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
import { LanguageEnum } from 'src/utils/languages.enum';
import {
AssignRoleToUserReq,
@@ -40,26 +44,19 @@ import {
GetAllHierarchiesRes,
GetAllJobTitlesRes,
GetAllUsersByCustomerIdRes,
IUserByCustomer,
SetUserRolesReq,
SetUserRolesRes,
UnassignRoleToUserReq,
UpdateUserReq,
UpdateUserRes,
IUserByCustomer,
} from './dtos/entities';
import { UsersService } from './users.service';
import ErrorBuilder from 'src/utils/ErrorBuilder';
import ErrorCodes from 'src/utils/errorCodes';
import { PackTheMetadata } from 'src/utils/ PackTheMetadata';
@ApiInternalOnlyController()
@ApiTags('Users')
@Controller('users')
@ApiHeader({
name: 'dadosfera-lang',
enum: LanguageEnum,
required: false,
description: 'Defaults to pt-br',
})
@ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }])
@Authenticated()
@UseFilters(GrpcToHttpExceptionFilter)
export class UsersController {
@@ -76,7 +73,7 @@ export class UsersController {
@Get()
async getAllUsersByCustomerId(
@User() user: RequestUser,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
): Promise<GetAllUsersByCustomerIdRes> {
this.logger.info('getAllUsersByCustomerId', { user });
this.userService.setLanguage(language);
@@ -88,7 +85,7 @@ export class UsersController {
@ApiOkResponse({ type: GetAllHierarchiesRes })
async getAllHierarchies(
@User() user: RequestUser,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
): Promise<GetAllHierarchiesRes> {
this.logger.info('getAllHierarchies', { user });
this.userService.setLanguage(language);
@@ -100,7 +97,7 @@ export class UsersController {
@ApiOkResponse({ type: GetAllDepartmentsRes })
async getAllDepartments(
@User() user: RequestUser,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
): Promise<GetAllDepartmentsRes> {
this.logger.info('getAllHierarchies', { user });
this.userService.setLanguage(language);
@@ -112,7 +109,7 @@ export class UsersController {
@ApiOkResponse({ type: GetAllDepartmentsRes })
async getAllJobTitles(
@User() user: RequestUser,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
): Promise<GetAllJobTitlesRes> {
this.logger.info('getAllHierarchies', { user });
this.userService.setLanguage(language);
@@ -126,7 +123,7 @@ export class UsersController {
async findOneById(
@User() user: RequestUser,
@Param('id') id: string,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('findOneById', { user });
this.userService.setLanguage(language);
@@ -147,7 +144,7 @@ export class UsersController {
async createUser(
@User() user: RequestUser,
@Body() body: CreateUserReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
const metadata = PackTheMetadata({
access_token: user.access_token,
@@ -165,7 +162,7 @@ export class UsersController {
async batchCreateUser(
@User() user: RequestUser,
@Body() body: BatchCreateUserReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('batchCreateUser', { user });
this.userService.setLanguage(language);
@@ -184,7 +181,7 @@ export class UsersController {
async resendInvite(
@User() user: RequestUser,
@Param('id') id: string,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('resendInvite', { user });
this.userService.setLanguage(language);
@@ -200,7 +197,7 @@ export class UsersController {
async unassignRoleToUser(
@User() user: RequestUser,
@Query() request: UnassignRoleToUserReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('unassignRoleToUser', { user });
this.userService.setLanguage(language);
@@ -213,7 +210,7 @@ export class UsersController {
async deleteUser(
@User() user: RequestUser,
@Param('id') id: string,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('deleteUser', { user });
this.userService.setLanguage(language);
@@ -226,7 +223,7 @@ export class UsersController {
async setUserRoles(
@User() user: RequestUser,
@Body() body: SetUserRolesReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('createUser', { user });
this.userService.setLanguage(language);
@@ -238,7 +235,7 @@ export class UsersController {
async assignRoleToUser(
@User() user: RequestUser,
@Body() body: AssignRoleToUserReq,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('assignRoleToUser', { user });
this.userService.setLanguage(language);
@@ -252,7 +249,7 @@ export class UsersController {
@User() user: RequestUser,
@Body() body: UpdateUserReq,
@Param('id') id: string,
@Headers('dadosfera-lang') language,
@Language() language: LanguageEnum,
) {
this.logger.info('updateUser', { user });
this.userService.setLanguage(language);
+22
View File
@@ -119,6 +119,14 @@ export function EnrichErrorCode(code: string) {
code,
};
case ErrorCodes.AUTH.INVALID_REFRESH_TOKEN:
return {
statusCode: HttpStatus.UNAUTHORIZED,
error: 'Acesso expirado!',
message: 'Seu acesso expirou, favor fazer login novamente.',
code,
};
case ErrorCodes.AUTH.FORBIDDEN:
return {
statusCode: HttpStatus.FORBIDDEN,
@@ -300,6 +308,20 @@ export function EnrichErrorCode(code: string) {
'Tente realizar a ação novamente. Caso o erro persista, entre em contato com o suporte',
code,
};
case ErrorCodes.CATALOG.PREVIEW_TOO_BIG:
return {
statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
error: 'Erro ao buscar prévia',
message: 'Ocorreu um erro devido ao tamanho da prévia do ativo.',
code,
};
case ErrorCodes.CATALOG.METADATA_TOO_BIG:
return {
statusCode: HttpStatus.INTERNAL_SERVER_ERROR,
error: 'Erro ao buscar informações',
message: 'Ocorreu um erro devido ao tamanho das informações do ativo.',
code,
};
case ErrorCodes.INTERNAL:
case ErrorCodes.UNKNOWN:
default:
+19 -1
View File
@@ -1,4 +1,7 @@
export function getObjValueFromPath<T>(data: { object: T; path: string }): T {
export function getObjValueFromPath<T extends object>(data: {
object: T;
path: string;
}): any {
const { object, path } = data;
const keys = path.split('.');
@@ -8,3 +11,18 @@ export function getObjValueFromPath<T>(data: { object: T; path: string }): T {
}
return value;
}
export function objHasPath<T extends object>(data: {
object: T;
path: string;
}): boolean {
const { object, path } = data;
const keys = path.split('.');
let value = JSON.parse(JSON.stringify(object));
for (const key of keys) {
if (!(key in value)) return false;
value = value[key];
}
return true;
}
+4
View File
@@ -1,5 +1,6 @@
export const AUTH = {
UNAUTHORIZED: 'AUTH.UNAUTHORIZED',
INVALID_REFRESH_TOKEN: 'AUTH.INVALID_REFRESH_TOKEN',
FORBIDDEN: 'AUTH.FORBIDDEN',
WRONG_CREDENTIALS: 'AUTH.WRONG_CREDENTIALS',
WRONG_PASSWORD_CONFIRMATION: 'AUTH.WRONG_PASSWORD_CONFIRMATION',
@@ -38,6 +39,7 @@ export const CUSTOMER = {
INVALID_ID: 'CUSTOMER.INVALID_ID',
NOT_FOUND: 'CUSTOMER.NOT_FOUND',
ALREADY_EXISTS: 'CUSTOMER.ALREADY_EXISTS',
BAD_REQUEST: 'CUSTOMER.BAD_REQUEST',
};
const TERMS_OF_USE = {
UP_TO_DATE: 'TERMS_OF_USE.UP_TO_DATE',
@@ -69,6 +71,8 @@ const TRANSFORMATION = {
};
const CATALOG = {
DATA_ASSET_NOT_FOUND: 'CATALOG.DATA_ASSET_NOT_FOUND',
PREVIEW_TOO_BIG: 'CATALOG.PREVIEW_TOO_BIG',
METADATA_TOO_BIG: 'CATALOG.METADATA_TOO_BIG',
};
const ErrorCodes = {
UNKNOWN: 'UNKNOWN',