diff --git a/Dockerfile b/Dockerfile index 79bae4a..eac9682 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,19 @@ FROM base_image AS build_base WORKDIR /app RUN apk update # needed packages to build dependencies from source -RUN apk add --no-cache aws-cli +RUN apk add --no-cache \ + aws-cli \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont COPY package*.json ./ +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + # run aws cli without mounting secret, because CI already has AWS credentials FROM build_base AS ci_image @@ -40,4 +50,17 @@ WORKDIR /app COPY --from=prod_build /app/dist ./dist COPY --from=prod_build /app/node_modules ./node_modules COPY --from=prod_build /app/package*.json ./ +RUN apk update +# needed packages to build dependencies from source +RUN apk add --no-cache \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont + +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + ENTRYPOINT npm run start:prod diff --git a/docsfera.json b/docsfera.json index 8f2b81f..b8d9671 100644 --- a/docsfera.json +++ b/docsfera.json @@ -20,6 +20,14 @@ ], "type": "string" } + }, + { + "name": "origin", + "required": true, + "in": "header", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -64,6 +72,14 @@ ], "type": "string" } + }, + { + "name": "origin", + "required": true, + "in": "header", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -249,6 +265,14 @@ ], "type": "string" } + }, + { + "name": "origin", + "required": true, + "in": "header", + "schema": { + "type": "string" + } } ], "responses": { @@ -4939,6 +4963,52 @@ ] } }, + "/catalog/pii-reporter": { + "get": { + "operationId": "CatalogController_getPiiReporter", + "parameters": [ + { + "name": "dadosfera-lang", + "in": "header", + "required": false, + "schema": { + "enum": [ + "pt-br", + "en-us" + ], + "type": "string" + } + }, + { + "name": "type", + "required": true, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Catalog" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/terms-of-use/token": { "get": { "operationId": "TermsOfUseController_getToken", @@ -5449,6 +5519,57 @@ ] } }, + "/customers/{id}/mfa": { + "post": { + "operationId": "CustomersController_enableMfaEnforce", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/EnforceMfa" + } + } + } + }, + "responses": { + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, "/customers/{id}/links": { "get": { "operationId": "CustomersController_getCustomerLinks", @@ -5596,6 +5717,34 @@ "Customers" ], "security": [ + { + "access-token": [] + }, + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } + }, + "/customers/access-dashboard": { + "get": { + "operationId": "CustomersController_getAccessDashboard", + "parameters": [], + "responses": { + "200": { + "description": "" + } + }, + "tags": [ + "Customers" + ], + "security": [ + { + "access-token": [] + }, { "access-token": [] }, @@ -5737,6 +5886,96 @@ "Health" ] } + }, + "/network-policy": { + "get": { + "operationId": "NetworkPolicyController_getNetworks", + "parameters": [], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NetworkPoliciesDTO" + } + } + } + } + }, + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "post": { + "operationId": "NetworkPolicyController_applyNetworkPolicies", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NetworkPoliciesDTO" + } + } + } + }, + "responses": { + "200": { + "description": "" + }, + "201": { + "description": "", + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + }, + "delete": { + "operationId": "NetworkPolicyController_removeNetworkPolicies", + "parameters": [], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/NetworkPoliciesDTO" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + }, + "security": [ + { + "access-token": [] + }, + { + "access-token": [] + } + ] + } } }, "info": { @@ -5807,6 +6046,9 @@ }, "themeEnabled": { "type": "boolean" + }, + "enforceMfa": { + "type": "boolean" } }, "required": [ @@ -5817,7 +6059,8 @@ "tier", "scheduleLimit", "links", - "themeEnabled" + "themeEnabled", + "enforceMfa" ] }, "AuthUser": { @@ -8486,6 +8729,17 @@ "tags" ] }, + "EnforceMfa": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ] + }, "CustomerLink": { "type": "object", "properties": { @@ -8588,6 +8842,20 @@ "required": [ "theme" ] + }, + "NetworkPoliciesDTO": { + "type": "object", + "properties": { + "policies": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "policies" + ] } } } diff --git a/helmfiles/stg.yaml b/helmfiles/stg.yaml index 530416b..5782e36 100644 --- a/helmfiles/stg.yaml +++ b/helmfiles/stg.yaml @@ -9,11 +9,11 @@ charts: - name: hostname value: maestro.stg.dadosfera.ai - name: maestro.pi_factory_url - value: pi-factory.stg.dadosfera.ai + value: pi-factory.dadosfera.ai - name: maestro.in_factory_url value: in-factory.stg.dadosfera.ai - name: maestro.tr_factory_url - value: in-factory.stg.dadosfera.ai + value: in-factory.dadosfera.ai - name: maestro.open_customer_id value: b3e3dfe5-b992-4586-a73c-c0b0c00f615d - name: maestro.open_group_id diff --git a/nest-cli.json b/nest-cli.json index 69ccf04..5085879 100644 --- a/nest-cli.json +++ b/nest-cli.json @@ -4,6 +4,7 @@ "compilerOptions": { "assets": [ "**/*.proto", + "assets/**/*", { "include": "i18n/**/*", "watchAssets": true diff --git a/package-lock.json b/package-lock.json index 20c2fcd..c9d852f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-alpha.19", + "@dadosfera/protospack-v2": "3.37.0-beta.23", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -30,8 +30,10 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cron-parser": "^4.9.0", + "csv": "^6.3.11", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", + "handlebars": "^4.7.8", "helmet": "^5.1.1", "jsonwebtoken": "^9.0.2", "jwk-to-pem": "^2.0.5", @@ -43,6 +45,7 @@ "passport-google-oauth20": "^2.0.0", "passport-hubspot-oauth2": "^1.0.3", "passport-mailchimp": "^1.1.0", + "puppeteer": "^24.7.2", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", "rxjs": "^7.5.5", @@ -1397,9 +1400,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "3.37.0-alpha.19", - "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-alpha.19.tgz", - "integrity": "sha512-+F2sMJszfXRqWuN5ae+v7MqHhJyg7TgVvnuFBKUOwHvBoWCfAUnIm9XGEA/YjPpRUa1gM0nnKL8SpBFf43Q1aA==", + "version": "3.37.0-beta.23", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz", + "integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", @@ -2945,6 +2948,27 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, + "node_modules/@puppeteer/browsers": { + "version": "2.10.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.2.tgz", + "integrity": "sha512-i4Ez+s9oRWQbNjtI/3+jxr7OH508mjAKvza0ekPJem0ZtmsYHP3B5dq62+IaBHKaGCOuqJxXzvFLUhJvQ6jtsQ==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.0", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.1", + "tar-fs": "^3.0.8", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@sinonjs/commons": { "version": "1.8.6", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz", @@ -3540,6 +3564,12 @@ "node": ">= 6" } }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", @@ -3985,6 +4015,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.62.0.tgz", @@ -4635,6 +4675,18 @@ "safer-buffer": "^2.1.0" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -4707,6 +4759,12 @@ "form-data": "^4.0.0" } }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, "node_modules/babel-jest": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.5.1.tgz", @@ -4813,6 +4871,78 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/bare-events": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz", + "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.1.3.tgz", + "integrity": "sha512-OeEZYIg+2qepaWLyphaOXHAHKo3xkM8y3BeGAvHdMN8GNWvEAU1Yw6rYpGzu/wDDbKxgEjVeVDpgGhDzaeMpjg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.1.tgz", + "integrity": "sha512-uaIjxokhFidJP+bmmvKSgiMzj2sV5GPHaZVAIktcxcpCyBFFWO+YlikVAdhmUo2vYFvFhOXIAlldqV29L8126g==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.5.tgz", + "integrity": "sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -4854,6 +4984,15 @@ "node": ">= 0.8" } }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -5057,6 +5196,15 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -5243,6 +5391,19 @@ "node": ">=6.0" } }, + "node_modules/chromium-bidi": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-4.1.1.tgz", + "integrity": "sha512-biR7t4vF3YluE6RlMSk9IWk+b9U+WWyzHp+N2pL9vRTk+UXHYRTVp7jTK58ZNzMLBgoLMHY4QyJMbeuw3eKxqg==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/ci-info": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", @@ -5687,6 +5848,48 @@ "dev": true, "license": "MIT" }, + "node_modules/csv": { + "version": "6.3.11", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.11.tgz", + "integrity": "sha512-a8bhT76Q546jOElHcTrkzWY7Py925mfLO/jqquseH61ThOebYwOjLbWHBqdRB4K1VpU36sTyIei6Jwj7QdEZ7g==", + "license": "MIT", + "dependencies": { + "csv-generate": "^4.4.2", + "csv-parse": "^5.6.0", + "csv-stringify": "^6.5.2", + "stream-transform": "^3.3.3" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.4.2.tgz", + "integrity": "sha512-W6nVsf+rz0J3yo9FOjeer7tmzBJKaTTxf7K0uw6GZgRocZYPVpuSWWa5/aoWWrjQZj4/oNIKTYapOM7hiNjVMA==", + "license": "MIT" + }, + "node_modules/csv-parse": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz", + "integrity": "sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==", + "license": "MIT" + }, + "node_modules/csv-stringify": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.2.tgz", + "integrity": "sha512-RFPahj0sXcmUyjrObAK+DOWtMvMIFV328n4qZJhgX3x2RqkQgOTU2mCUmiFR0CzM6AzChlRSUErjiJeEt8BaQA==", + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/data-urls": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", @@ -5801,6 +6004,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -5839,6 +6056,12 @@ "node": ">=8" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1425554", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz", + "integrity": "sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==", + "license": "BSD-3-Clause" + }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", @@ -6141,6 +6364,15 @@ "node": ">=10.13.0" } }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/error-callsites": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/error-callsites/-/error-callsites-2.0.4.tgz", @@ -6250,7 +6482,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "esprima": "^4.0.1", @@ -6272,7 +6503,6 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=4.0" @@ -6282,7 +6512,6 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, "license": "BSD-3-Clause", "optional": true, "engines": { @@ -6493,7 +6722,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, "license": "BSD-2-Clause", "bin": { "esparse": "bin/esparse.js", @@ -6560,7 +6788,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=0.10.0" @@ -6790,6 +7017,41 @@ "node": ">=4" } }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -6803,6 +7065,12 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -6899,6 +7167,15 @@ "bser": "2.1.1" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/fecha": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", @@ -7272,6 +7549,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-uri": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.4.tgz", + "integrity": "sha512-E1b1lFFLvLgak2whF2xDBcOy6NLVGZBqqjJjsIhvopKfWWEi64pLVTWWehV8KlLerZkfNTA95sTe2OdJKm1OzQ==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -7373,6 +7664,36 @@ "dev": true, "license": "MIT" }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/handlebars/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -7724,6 +8045,25 @@ "node": ">= 0.10" } }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "license": "MIT", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -8722,6 +9062,12 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", + "license": "MIT" + }, "node_modules/jsdom": { "version": "16.7.0", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", @@ -9411,6 +9757,12 @@ "node": ">=8" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/mixpanel": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/mixpanel/-/mixpanel-0.17.0.tgz", @@ -9522,6 +9874,15 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "license": "MIT" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/next-line": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", @@ -9875,6 +10236,73 @@ "node": ">=6" } }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -10155,6 +10583,12 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -10372,6 +10806,15 @@ "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==", "license": "MIT" }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -10433,6 +10876,75 @@ "node": ">= 0.10" } }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -10471,6 +10983,91 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "24.7.2", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.7.2.tgz", + "integrity": "sha512-ifYqoY6wGs0yZeFuFPn8BE9FhuveXkarF+eO18I2e/axdoCh4Qh1AE+qXdJBhdaeoPt6eRNTY4Dih29Jbq8wow==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.10.2", + "chromium-bidi": "4.1.1", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1425554", + "puppeteer-core": "24.7.2", + "typed-query-selector": "^2.12.0" + }, + "bin": { + "puppeteer": "lib/cjs/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core": { + "version": "24.7.2", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.7.2.tgz", + "integrity": "sha512-P9pZyTmJqKODFCnkZgemCpoFA4LbAa8+NumHVQKyP5X9IgdNS1ZnAnIh1sMAwhF8/xEUGf7jt+qmNLlKieFw1Q==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.10.2", + "chromium-bidi": "4.1.1", + "debug": "^4.4.0", + "devtools-protocol": "0.0.1425554", + "typed-query-selector": "^2.12.0", + "ws": "^8.18.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/puppeteer/node_modules/cosmiconfig": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", + "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/qs": { "version": "6.11.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", @@ -11148,6 +11745,53 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.4.tgz", + "integrity": "sha512-D3YaD0aRxR3mEcqnidIs7ReYJFVzWdd6fXJYUM8ixcQcJRGTka/b3saV0KflYhyVJXKhb947GndU35SxYNResQ==", + "license": "MIT", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks-proxy-agent/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/sonic-boom": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", @@ -11255,6 +11899,12 @@ "readable-stream": "^3.0.6" } }, + "node_modules/stream-transform": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.3.tgz", + "integrity": "sha512-dALXrXe+uq4aO5oStdHKlfCM/b3NBdouigvxVPxCdrMRAU6oHh3KNss20VbTPQNQmjAHzZGKGe66vgwegFEIog==", + "license": "MIT" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -11263,6 +11913,19 @@ "node": ">=10.0.0" } }, + "node_modules/streamx": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.0.tgz", + "integrity": "sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -11510,6 +12173,31 @@ "node": ">=6" } }, + "node_modules/tar-fs": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz", + "integrity": "sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/terminal-link": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", @@ -11619,6 +12307,15 @@ "node": ">=8" } }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -12034,6 +12731,12 @@ "node": ">= 0.6" } }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", + "license": "MIT" + }, "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", @@ -12063,6 +12766,19 @@ "node": ">=4.2.0" } }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/uid": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz", @@ -12558,6 +13274,12 @@ "node": ">=0.10.0" } }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "license": "MIT" + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -12690,6 +13412,16 @@ "node": ">=12" } }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -12712,6 +13444,15 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.24.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.3.tgz", + "integrity": "sha512-HhY1oqzWCQWuUqvBFnsyrtZRhyPeR7SUGv+C4+MsisMuVfSPx8HpwWqH8tRahSlt6M3PiFAcoeFhZAqIXTxoSg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index 91ed2f0..fd3eb0b 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "3.37.0-alpha.19", + "@dadosfera/protospack-v2": "3.37.0-beta.23", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -48,8 +48,10 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cron-parser": "^4.9.0", + "csv": "^6.3.11", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", + "handlebars": "^4.7.8", "helmet": "^5.1.1", "jsonwebtoken": "^9.0.2", "jwk-to-pem": "^2.0.5", @@ -61,6 +63,7 @@ "passport-google-oauth20": "^2.0.0", "passport-hubspot-oauth2": "^1.0.3", "passport-mailchimp": "^1.1.0", + "puppeteer": "^24.7.2", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", "rxjs": "^7.5.5", diff --git a/src/app.module.ts b/src/app.module.ts index 3e54fde..fedb388 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -28,6 +28,7 @@ import { MixpanelModule } from './modules/mixpanel/mixpanel.module'; import { CustomersModule } from './modules/customers/customers.module'; import { OpenDataModule } from './modules/open-data/open-data.module'; import { ThemeModule } from './modules/theme/theme.module'; +import { NetworkPolicyModule } from './modules/network-policy/network-policy.module'; @Module({ providers: [ @@ -62,7 +63,8 @@ import { ThemeModule } from './modules/theme/theme.module'; OpenDataModule, ThemeModule, //Always leave HealthModule last, so it is on the bottom of swagger - HealthModule + HealthModule, + NetworkPolicyModule ], }) export class AppModule {} diff --git a/src/assets/template/pii-reporter.html b/src/assets/template/pii-reporter.html new file mode 100644 index 0000000..fb51aac --- /dev/null +++ b/src/assets/template/pii-reporter.html @@ -0,0 +1,130 @@ + + + + + + Dadosfera Relatório de PII + + + + +
+
+ +

Relatório de PII

+
+ +
+

Este relatório apresenta a estrutura de tabelas e suas as seguintes características de PII identificadas.

+
+ + + + + + + + + + + + + + {{#each dados}} + + + + + + + + + {{/each}} + +
DatabaseSchemaTabelaColunaTipo de DadoRegras PII
{{database_name}}{{table_schema}}{{table_name}}{{column_name}}{{data_type}}{{pii_rules}}
+ +

Gerado em: {{dataGeracao}}

+
+ + \ No newline at end of file diff --git a/src/authentication/permissions.enum.ts b/src/authentication/permissions.enum.ts index 1e01531..ee74e22 100644 --- a/src/authentication/permissions.enum.ts +++ b/src/authentication/permissions.enum.ts @@ -612,7 +612,9 @@ export interface DadosferaModule { export const DADOSFERA_MODULES_KEYS = { LOG_DASHBOARD: 'logs-dashboard', - ACCESS_DASHBOARD: 'access-dashboard' + ACCESS_DASHBOARD: 'access-dashboard', + DANGER_ZONE: 'danger-zone', + PII: 'pii' } export const DADOSFERA_MODULES: Array = [ diff --git a/src/modules/auth/auth.controller.ts b/src/modules/auth/auth.controller.ts index 65374a1..226036a 100644 --- a/src/modules/auth/auth.controller.ts +++ b/src/modules/auth/auth.controller.ts @@ -54,7 +54,6 @@ import { LanguageEnum } from 'src/utils/languages.enum'; import { Language } from 'src/decorators/language.decorator'; import { ApiInternalOnlyEndpoint } from 'src/decorators/swagger.decorator'; - @ApiTags('Auth') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @UseFilters(new GrpcToHttpExceptionFilter()) @@ -63,7 +62,6 @@ export class AuthController { logger: DadosferaLogger; redirectUrl: string; - constructor( @Inject(DadosferaLogger) dadosferaLogger: DadosferaLogger, @@ -88,9 +86,12 @@ export class AuthController { async signIn( @Body() { username, password, totp }: AuthSignInReq, @Language() language: LanguageEnum, + @Headers('origin') origin: string, ): Promise { this.logger.info('/auth - SignIn'); - const metadata = PackTheMetadata({ language }); + const frontHost = origin.replace(/^https?:\/\//, ''); + const metadata = PackTheMetadata({ language, custom_host: frontHost }); + this.logger.info('metadata: ' + JSON.stringify(metadata.toJSON())); return this.authClient.signIn({ username, password, totp }, metadata); } @@ -100,13 +101,15 @@ export class AuthController { async refreshAccessToken( @Body() body: AuthRefreshAccessTokenReq, @Language() language: LanguageEnum, + @Headers('origin') origin: string, ) { this.logger.info('/auth - RefreshAccessToken'); const { refreshToken, customerName: customer_name } = body; - + const frontHost = origin.replace(/^https?:\/\//, ''); const metadata = PackTheMetadata({ - customer_name, language, + customer_name, + custom_host: frontHost, }); return this.authClient.refreshAccessToken({ refreshToken }, metadata); @@ -180,16 +183,23 @@ export class AuthController { @ApiInternalOnlyEndpoint() @Post('confirm-reset-password') @HttpCode(HttpStatus.OK) - async confirmResetPassword(@Body() body: AuthConfirmResetPasswordRequest) { + async confirmResetPassword( + @Body() body: AuthConfirmResetPasswordRequest, + @Headers('origin') origin: string, + ) { this.logger.info('/auth - confirm-reset-password'); - + const frontHost = origin.replace(/^https?:\/\//, ''); + const metadata = PackTheMetadata({ custom_host: frontHost }); const { username, code, newPassword } = body; - return this.authClient.confirmResetPassword({ - username, - code, - newPassword, - }); + return this.authClient.confirmResetPassword( + { + username, + code, + newPassword, + }, + metadata, + ); } @ApiInternalOnlyEndpoint() @@ -355,18 +365,18 @@ export class AuthController { async blockUsers( @Language() language: LanguageEnum, @User() user: RequestUser, - @Body() body: BulkEditRequest + @Body() body: BulkEditRequest, ) { this.logger.info('blockUsers - Starting request'); - + try { const metadata = PackTheMetadata(user); this.logger.debug('Calling blockUsers service', { metadata: { access_token: metadata.get('access_token'), - language: metadata.get('language') - } + language: metadata.get('language'), + }, }); const result = await this.authClient.blockUsers(body.users, metadata); @@ -375,7 +385,7 @@ export class AuthController { } catch (error) { this.logger.error('blockUsers - Error', { error: error.message, - stack: error.stack + stack: error.stack, }); throw error; } @@ -388,7 +398,7 @@ export class AuthController { async unblockUsers( @Language() language: LanguageEnum, @User() user: RequestUser, - @Body() body: BulkEditRequest + @Body() body: BulkEditRequest, ) { this.logger.info('unblockUsers'); const metadata = PackTheMetadata(user); @@ -403,11 +413,11 @@ export class AuthController { async resetUsers( @Language() language: LanguageEnum, @User() user: RequestUser, - @Body() body: BulkEditRequest + @Body() body: BulkEditRequest, ) { this.logger.info('resetUsers'); const metadata = PackTheMetadata(user); - return this.authClient.resetUsers(body.users, metadata); + return this.authClient.resetUsers(body.users, metadata); } } diff --git a/src/modules/auth/auth.service.ts b/src/modules/auth/auth.service.ts index f99ff3a..5974954 100644 --- a/src/modules/auth/auth.service.ts +++ b/src/modules/auth/auth.service.ts @@ -113,19 +113,21 @@ export class AuthClientService implements OnModuleInit { ); } - async confirmResetPassword({ - username, - code, - newPassword, - }: AuthConfirmResetPasswordRequest) { + async confirmResetPassword( + { username, code, newPassword }: AuthConfirmResetPasswordRequest, + metadata: Metadata, + ) { this.logger.info('confirmResetPassword'); return lastValueFrom( - this.authService.AuthConfirmResetPassword({ - username, - code, - newPassword, - }), + this.authService.AuthConfirmResetPassword( + { + username, + code, + newPassword, + }, + metadata, + ), ); } @@ -170,15 +172,18 @@ export class AuthClientService implements OnModuleInit { ); } - async blockUsers(users: string[],metadata: Metadata): Promise { + async blockUsers( + users: string[], + metadata: Metadata, + ): Promise { this.logger.info('blockUsers - Service starting'); - + try { this.logger.debug('Calling BlockUser gRPC method', { metadata: { access_token: metadata.get('access_token'), - language: metadata.get('language') - } + language: metadata.get('language'), + }, }); const response = await lastValueFrom( @@ -196,26 +201,32 @@ export class AuthClientService implements OnModuleInit { } } - async unblockUsers(users: string[],metadata: Metadata): Promise { + async unblockUsers( + users: string[], + metadata: Metadata, + ): Promise { this.logger.info('unblockUsers'); return await lastValueFrom( this.authService.UnblockUser({ users }, metadata), ); } - async resetUsers(users: string[], metadata: Metadata): Promise { + async resetUsers( + users: string[], + metadata: Metadata, + ): Promise { this.logger.info('resetUsers'); - + try { this.logger.debug('Calling ResetUser gRPC method', { metadata: { access_token: metadata.get('access_token'), - language: metadata.get('language') - } + language: metadata.get('language'), + }, }); const response = await lastValueFrom( - this.authService.ResetUser({ users }, metadata) + this.authService.ResetUser({ users }, metadata), ); this.logger.info('resetUsers - Success', { response }); @@ -223,7 +234,7 @@ export class AuthClientService implements OnModuleInit { } catch (error) { this.logger.error('resetUsers - Error', { error: error.message, - stack: error.stack + stack: error.stack, }); throw error; } diff --git a/src/modules/auth/dtos/login.ts b/src/modules/auth/dtos/login.ts index b3890fe..0cd90ae 100644 --- a/src/modules/auth/dtos/login.ts +++ b/src/modules/auth/dtos/login.ts @@ -87,6 +87,8 @@ export class AuthCustomer { @ApiProperty() themeEnabled: boolean; + @ApiProperty() + enforceMfa: boolean; } export class AuthSignInReq implements AuthSignInRequest { diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 5f2d194..09daa16 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -15,6 +15,7 @@ import { UseFilters, HttpException, HttpStatus, + Res, } from '@nestjs/common'; import { ApiCreatedResponse, @@ -25,14 +26,14 @@ import { import { Authenticated, RequireAllPermissions, + RequireModule, RequireSomePermission, } from '../../decorators/authentication.decorator'; -import { PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; +import { DADOSFERA_MODULES_KEYS, PERMISSIONS_GROUPS } from '../../authentication/permissions.enum'; import { CatalogService } from './catalog.service'; import { DadosferaLogger } from '@dadosfera/dadosfera-logs'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; import { RequestUser, User } from 'src/decorators/user.decorator'; -import { lastValueFrom } from 'rxjs'; import { BatchRemoveRlsRulesRequest, GetDatasetCatalogTaskRes, @@ -59,6 +60,8 @@ import { GetRlsRulesRequest, RegisterDatasetWithMetatadaRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; +import { Response } from 'express'; +import { TypeParser } from 'src/utils/FileParser/parser-types'; @ApiTags('Catalog') @ApiHeaders([{ name: 'dadosfera-lang', enum: LanguageEnum, required: false }]) @@ -344,7 +347,7 @@ export class CatalogController { @Language() language: LanguageEnum, @Param('id') id: string, ): Promise { - const { customer_name, customer_id, user_id, username } = user; + const { customer_name, customer_id, user_id, username, customer_modules } = user; this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { user_id, @@ -357,6 +360,7 @@ export class CatalogController { user_id, username, language, + is_mask: customer_modules.some(mod => mod === 'pii') }); const preview = await this.catalogService.getDatasetPreview(id, metadata); @@ -748,25 +752,23 @@ export class CatalogController { return JSON.parse(dashboards); } - @Post('register-dataset') @RequireSomePermission( PERMISSIONS_GROUPS.CATALOG.permissions.CREATE, - PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER + PERMISSIONS_GROUPS.CATALOG.permissions.DATA_MANAGER, ) async registerDatasetWithMetadataRequest( @Body() body: RegisterDatasetWithMetatadaRequest, @User() user: RequestUser, ) { - const { customer_id, customer_name, user_id, username } = user; - const logMetadata = { - customer_name: customer_name, - user_id: user_id, - method: 'POST', - path: '/catalog/register-dataset', - }; + const { customer_id, customer_name, user_id, username } = user; + const logMetadata = { + customer_name: customer_name, + user_id: user_id, + method: 'POST', + path: '/catalog/register-dataset', + }; try { - const metadata = PackTheMetadata({ customer_id, customer_name, @@ -774,18 +776,20 @@ export class CatalogController { username, }); - this.logger.log(`Request from user ${user_id} for customer ${customer_name}`, logMetadata); + this.logger.log( + `Request from user ${user_id} for customer ${customer_name}`, + logMetadata, + ); // Create table metadata const tableMetadataBody = { table_metadata: body.table_metadata, info: { customer: customer_name, - }, - logMetadata: logMetadata + logMetadata: logMetadata, }; - + const table_metadata_id = await this.catalogService.createTableMetadata( tableMetadataBody, ); @@ -798,8 +802,12 @@ export class CatalogController { customer: customer_name, }, }; - this.logger.info(`Creating column metadata for table ${table_metadata_id}`, logMetadata); - const column_metadata_ids = await this.catalogService.createColumnMetadata(columnMetadataBody) + this.logger.info( + `Creating column metadata for table ${table_metadata_id}`, + logMetadata, + ); + const column_metadata_ids = + await this.catalogService.createColumnMetadata(columnMetadataBody); // Create data preview const dataPreviewBody = { @@ -808,15 +816,24 @@ export class CatalogController { customer: customer_name, }, }; - this.logger.debug(`Creating data preview for table ${table_metadata_id}`, logMetadata); + this.logger.debug( + `Creating data preview for table ${table_metadata_id}`, + logMetadata, + ); const data_preview_id = await this.catalogService.createDataPreview( dataPreviewBody, ); // Catalog dataset item - this.logger.info(`Cataloging dataset item for table ${table_metadata_id}`, logMetadata); + this.logger.info( + `Cataloging dataset item for table ${table_metadata_id}`, + logMetadata, + ); await this.catalogService.catalogDatasetItem(table_metadata_id, metadata); - this.logger.info(`Dataset registration completed successfully for table ${table_metadata_id}`, logMetadata); + this.logger.info( + `Dataset registration completed successfully for table ${table_metadata_id}`, + logMetadata, + ); return { message: 'Dataset registered successfully', table_metadata_id: table_metadata_id, @@ -824,7 +841,10 @@ export class CatalogController { data_preview_id: data_preview_id, }; } catch (error) { - this.logger.error(`Failed to register dataset. The following error occurred: ${error.response.data}`, logMetadata); + this.logger.error( + `Failed to register dataset. The following error occurred: ${error.response.data}`, + logMetadata, + ); throw new HttpException( { @@ -837,4 +857,32 @@ export class CatalogController { ); } } + + @Get('pii-reporter') + @RequireSomePermission( + PERMISSIONS_GROUPS.USERS.permissions.ADMIN + ) + @RequireModule( + DADOSFERA_MODULES_KEYS.PII + ) + async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') contentType: TypeParser = "pdf") { + this.logger.info('GET pii-reporter'); + const metadata = PackTheMetadata(user); + + try { + const { + file, + filename, + type + } = await this.catalogService.getPiiReporter(metadata, contentType); + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + res.setHeader('Content-Type', type); + + // use res.end to send buffer + return res.end(file); + } catch (error) { + console.error(error) + this.logger.error(error.message); + } + } } diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index adfa0a2..a3db9cc 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -33,7 +33,16 @@ import { AddRlsRuleRequest, GetNimbusDashboardsRequest, GetRlsRulesRequest, + PiiMetadata, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; +import { writeFileSync } from 'fs'; +import path from 'path'; +import { HtmlParser } from 'src/utils/FileParser/html-parser'; +import { PiiDto } from './dtos/pii.dto'; +import { CsvParser } from 'src/utils/FileParser/csv-parser'; +import { PDFParser } from 'src/utils/FileParser/pdf-parser'; +import { TypeParser } from 'src/utils/FileParser/parser-types'; +import { ParserBuilder } from 'src/utils/FileParser/parser.builder'; class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; @@ -80,6 +89,42 @@ class CatalogService implements OnModuleInit { )}.dadosfera.ai`; } + async getPiiReporter(metadata: Metadata, type: TypeParser) { + this.logger.info('getPiiReporter: ' + type) + try { + const { + data + } = await lastValueFrom( + this.catalogWriteService.GetPiiReporter({}, metadata) + ) + this.logger.info("Finish grpc call") + + const parser = ParserBuilder.build(type); + + this.logger.info('parser file to: ' + type) + const file = await parser.parse(data) + this.logger.info('finish parser') + const mimeTypes: Record = { + 'csv': 'text/csv', + 'html': 'text/html', + 'pdf': 'application/pdf' + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `relatorio-pii-${timestamp}.${type}`; + + return { + file, + filename: filename, + type: mimeTypes[type] + } + } catch (error) { + this.logger.error(error.message); + throw error; + } + + } + async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { this.logger.info('CatalogService - Manage Data assets permissions'); if (!data.embed) data.embed = undefined; diff --git a/src/modules/catalog/dtos/pii.dto.ts b/src/modules/catalog/dtos/pii.dto.ts new file mode 100644 index 0000000..c456b2b --- /dev/null +++ b/src/modules/catalog/dtos/pii.dto.ts @@ -0,0 +1,8 @@ +export class PiiDto { + database_name: string; + table_schema: string; + table_name: string; + column_name: string; + data_type: string; + pii_rules: string; +} diff --git a/src/modules/customers/customers.controller.ts b/src/modules/customers/customers.controller.ts index 280dce3..bcab557 100644 --- a/src/modules/customers/customers.controller.ts +++ b/src/modules/customers/customers.controller.ts @@ -7,6 +7,7 @@ import { HttpStatus, Inject, Param, + Post, Put, Query, UseFilters, @@ -17,6 +18,7 @@ import { Authenticated, RequireAllPermissions, RequireModule, + RequireSomePermission, } from 'src/decorators/authentication.decorator'; import { GrpcToHttpExceptionFilter } from 'src/error/grpc-to-http-exception.filter'; import { CustomersService } from './customers.service'; @@ -24,6 +26,7 @@ import { CustomerLinkRequest, CustomerLinksResponse } from './dtos/customers'; import { RequestUser, User } from 'src/decorators/user.decorator'; import type { StringValue } from 'ms'; import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; +import { EnforceMfa } from './dtos/enforce-mfa'; @ApiTags('Customers') @Controller('customers') @@ -39,6 +42,15 @@ export class CustomersController { this.logger = dadosferaLogger.logger; } + @Post(':id/mfa') + @Authenticated() + @RequireSomePermission(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @RequireModule(DADOSFERA_MODULES_KEYS.DANGER_ZONE) + async enableMfaEnforce(@Param('id') id: string, @Body() data: EnforceMfa) { + this.logger.info('enableMfaEnforce', { id }); + return await this.customersService.enableEnforceMfa(id, data.enabled); + } + @Get(':id/links') @Authenticated() @ApiOkResponse({ type: CustomerLinksResponse }) diff --git a/src/modules/customers/customers.service.ts b/src/modules/customers/customers.service.ts index ed302fd..056c5fb 100644 --- a/src/modules/customers/customers.service.ts +++ b/src/modules/customers/customers.service.ts @@ -34,6 +34,7 @@ import DadosferaLogger from '@dadosfera/dadosfera-logs'; // This function will accept any string, which may result in a bug. @Injectable() export class CustomersService implements OnModuleInit { + private customerService: CustomersProtoService; private logger: DadosferaLogger; private pipelineReadService: ReadService.PipelineV2ReadService; @@ -203,4 +204,13 @@ export class CustomersService implements OnModuleInit { ) ); } + + async enableEnforceMfa(id: string, enabled: boolean) { + return await lastValueFrom( + this.customerService.CustomerUpdateEnforceMfa({ + customerId: id, + enforceMfa: enabled + }) + ) + } } diff --git a/src/modules/customers/dtos/enforce-mfa.ts b/src/modules/customers/dtos/enforce-mfa.ts new file mode 100644 index 0000000..4709579 --- /dev/null +++ b/src/modules/customers/dtos/enforce-mfa.ts @@ -0,0 +1,6 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class EnforceMfa { + @ApiProperty() + enabled: boolean +} \ No newline at end of file diff --git a/src/modules/network-policy/dto/network-policy.dto.ts b/src/modules/network-policy/dto/network-policy.dto.ts new file mode 100644 index 0000000..ec4f6c2 --- /dev/null +++ b/src/modules/network-policy/dto/network-policy.dto.ts @@ -0,0 +1,6 @@ +import { ApiProperty } from "@nestjs/swagger"; + +export class NetworkPoliciesDTO { + @ApiProperty() + policies: string [] +} \ No newline at end of file diff --git a/src/modules/network-policy/network-policy.controller.ts b/src/modules/network-policy/network-policy.controller.ts new file mode 100644 index 0000000..8ae8371 --- /dev/null +++ b/src/modules/network-policy/network-policy.controller.ts @@ -0,0 +1,64 @@ +import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Post } from '@nestjs/common'; +import { ApiOkResponse } from '@nestjs/swagger'; +import { PERMISSIONS_GROUPS } from 'src/authentication/permissions.enum'; +import { + Authenticated, + RequireAllPermissions, +} from 'src/decorators/authentication.decorator'; +import { NetworkPoliciesDTO } from './dto/network-policy.dto'; +import { RequestUser, User } from 'src/decorators/user.decorator'; +import { NetworkPolicyService } from './network-policy.service'; +import { PackTheMetadata } from 'src/utils/ PackTheMetadata'; + +@Controller('network-policy') +export class NetworkPolicyController { + constructor(private networkPolicyService: NetworkPolicyService) {} + + @Get() + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @HttpCode(HttpStatus.OK) + async getNetworks( + @User() user: RequestUser, + ) { + return await this.networkPolicyService.getByCustomer( + user.customer_id + ); + } + + @Post() + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @ApiOkResponse() + @HttpCode(HttpStatus.CREATED) + async applyNetworkPolicies( + @User() user: RequestUser, + @Body() data: NetworkPoliciesDTO, + ) { + const metadata = PackTheMetadata(user); + + return await this.networkPolicyService.apply( + data.policies, + user.customer_id, + metadata, + ); + } + + @Delete() + @Authenticated() + @RequireAllPermissions(PERMISSIONS_GROUPS.USERS.permissions.ADMIN) + @ApiOkResponse() + @HttpCode(HttpStatus.OK) + async removeNetworkPolicies( + @User() user: RequestUser, + @Body() data: NetworkPoliciesDTO, + ) { + const metadata = PackTheMetadata(user); + + return await this.networkPolicyService.delete( + data.policies, + user.customer_id, + metadata, + ); + } +} diff --git a/src/modules/network-policy/network-policy.module.ts b/src/modules/network-policy/network-policy.module.ts new file mode 100644 index 0000000..34e2e36 --- /dev/null +++ b/src/modules/network-policy/network-policy.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common'; +import { NetworkPolicyController } from './network-policy.controller'; +import { NetworkPolicyService } from './network-policy.service'; +import { ClientsModule } from '@nestjs/microservices'; +import { DucClient } from '../duc/client.config'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; + +const ducClient = new DucClient(); + +@Module({ + imports: [ + ClientsModule.register([ + ducClient.providerOptions + ]), + ], + controllers: [NetworkPolicyController], + providers: [NetworkPolicyService, DadosferaLogger] +}) +export class NetworkPolicyModule {} diff --git a/src/modules/network-policy/network-policy.service.ts b/src/modules/network-policy/network-policy.service.ts new file mode 100644 index 0000000..ed39915 --- /dev/null +++ b/src/modules/network-policy/network-policy.service.ts @@ -0,0 +1,74 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { DucClient } from '../duc/client.config'; +import { ClientGrpc } from '@nestjs/microservices'; +import DadosferaLogger from '@dadosfera/dadosfera-logs'; +import { CustomersProtoService } from '@dadosfera/protospack-v2/dist/lib/Duc/interfaces/write-service'; +import { ProtoServices } from '@dadosfera/protospack-v2/dist/lib/Duc'; +import { Metadata } from '@grpc/grpc-js'; +import { lastValueFrom } from 'rxjs'; +import { NetworkPoliciesDTO } from './dto/network-policy.dto'; + + +@Injectable() +export class NetworkPolicyService { + private customerService: CustomersProtoService; + private logger: DadosferaLogger; + + constructor( + @Inject(DucClient.name) private readonly grpcClient: ClientGrpc, + @Inject(DadosferaLogger) + dadosferaLogger: DadosferaLogger, + ) { + this.logger = dadosferaLogger.logger; + } + + onModuleInit() { + this.customerService = this.grpcClient.getService( + ProtoServices.CustomersProtoService, + ); + } + + async getByCustomer(id: string): Promise { + const { + customer + } = await lastValueFrom(this.customerService.CustomerFindOneById({ + id + })); + + return { + policies: customer.networkPolicies + } + } + + async apply( + networkPolicies: string[], + customerId: string, + metadata: Metadata, + ) { + return await lastValueFrom( + this.customerService.CustomerCreateNetworkPolicy( + { + customerId, + networkPolicies, + }, + metadata, + ), + ); + } + + async delete( + networkPolicies: string[], + customerId: string, + metadata: Metadata, + ) { + return await lastValueFrom( + this.customerService.CustomerRemoveNetworkPolicy( + { + customerId, + networkPolicies, + }, + metadata, + ), + ); + } +} diff --git a/src/utils/ PackTheMetadata.ts b/src/utils/ PackTheMetadata.ts index 23556ea..18958fe 100644 --- a/src/utils/ PackTheMetadata.ts +++ b/src/utils/ PackTheMetadata.ts @@ -11,6 +11,10 @@ interface IMetadata { roles?: string[]; is_data_manager?: boolean; access_token?: string; + host?: string; + origin?: string; + custom_host?: string; + is_mask?: boolean; } export function PackTheMetadata(info: IMetadata): Metadata { diff --git a/src/utils/FileParser/csv-parser.ts b/src/utils/FileParser/csv-parser.ts new file mode 100644 index 0000000..c481710 --- /dev/null +++ b/src/utils/FileParser/csv-parser.ts @@ -0,0 +1,14 @@ +import { stringify } from 'csv' +import { Parser } from './parser.interface'; + +export class CsvParser implements Parser{ + async parse(data: T[]) { + return new Promise((resolve, reject) => { + stringify(data, { header: true }, function (err, output) { + if (err) reject(err); + + resolve(output) + }); + }) + } +} diff --git a/src/utils/FileParser/html-parser.ts b/src/utils/FileParser/html-parser.ts new file mode 100644 index 0000000..0e7f155 --- /dev/null +++ b/src/utils/FileParser/html-parser.ts @@ -0,0 +1,17 @@ +import { readFileSync } from 'fs'; +import Handlebars from 'handlebars'; +import { Parser } from './parser.interface'; + +export class HtmlParser implements Parser { + constructor(private readonly templatePath: string) {} + async parse(data: T[]) { + const templateSource = readFileSync(this.templatePath, 'utf8'); + + const template = Handlebars.compile(templateSource); + + return template({ + dados: data, + dataGeracao: new Date().toLocaleString('pt-BR'), + }); + } +} diff --git a/src/utils/FileParser/parser-types.ts b/src/utils/FileParser/parser-types.ts new file mode 100644 index 0000000..021ac28 --- /dev/null +++ b/src/utils/FileParser/parser-types.ts @@ -0,0 +1 @@ +export type TypeParser = "html" | "csv" | "pdf" \ No newline at end of file diff --git a/src/utils/FileParser/parser.builder.ts b/src/utils/FileParser/parser.builder.ts new file mode 100644 index 0000000..7e2d5d9 --- /dev/null +++ b/src/utils/FileParser/parser.builder.ts @@ -0,0 +1,25 @@ +import { TypeParser } from './parser-types'; +import { HtmlParser } from './html-parser'; +import { CsvParser } from './csv-parser'; +import { PDFParser } from './pdf-parser'; +import { Parser } from './parser.interface'; + +export class ParserBuilder { + static build(type: TypeParser): Parser { + const templatePath = '/app/dist/assets/template/pii-reporter.html'; + + if (type === 'html') { + const htmlParser = new HtmlParser(templatePath); + return htmlParser; + } else if (type === 'csv') { + const csvParser = new CsvParser(); + return csvParser; + } else if (type === 'pdf') { + const htmlParser = new HtmlParser(templatePath); + const pdfParser = new PDFParser(htmlParser); + return pdfParser; + } + + throw new Error('Unsupported Type: ' + type); + } +} diff --git a/src/utils/FileParser/parser.interface.ts b/src/utils/FileParser/parser.interface.ts new file mode 100644 index 0000000..26af780 --- /dev/null +++ b/src/utils/FileParser/parser.interface.ts @@ -0,0 +1,3 @@ +export interface Parser { + parse(data: T[]): Promise +} \ No newline at end of file diff --git a/src/utils/FileParser/pdf-parser.ts b/src/utils/FileParser/pdf-parser.ts new file mode 100644 index 0000000..1294b34 --- /dev/null +++ b/src/utils/FileParser/pdf-parser.ts @@ -0,0 +1,57 @@ +import puppeteer from 'puppeteer'; +import { Parser } from './parser.interface'; +import { HtmlParser } from './html-parser'; + +export class PDFParser implements Parser { + constructor(private htmlParser: HtmlParser) {} + + async parse(data: T[]) { + try { + const browser = await puppeteer.launch({ + executablePath: process.env.PUPPETEER_EXECUTABLE_PATH, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + ], + }); + + const page = await browser.newPage(); + page.setDefaultNavigationTimeout(0); + + const html = await this.htmlParser.parse(data); + await page.setContent(html, { + waitUntil: 'networkidle0', + timeout: 30000, + }); + + // Configurações adicionais para garantir um PDF válido + const pdf = await page.pdf({ + format: 'A4', + printBackground: true, + margin: { + top: '20mm', + right: '15mm', + bottom: '20mm', + left: '15mm', + }, + preferCSSPageSize: true, + displayHeaderFooter: false, + headerTemplate: '', + footerTemplate: '', + omitBackground: false, + pageRanges: '', + tagged: true, + outline: false, + timeout: 30000, + }); + + await browser.close(); + + return Buffer.from(pdf); + } catch (error) { + console.error('Error generating PDF:', error); + throw error; + } + } +}