diff --git a/Dockerfile b/Dockerfile index 79bae4a..3186c71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,9 +4,19 @@ FROM base_image AS build_base WORKDIR /app RUN apk update # needed packages to build dependencies from source -RUN apk add --no-cache aws-cli +RUN apk add --no-cache \ + aws-cli \ + chromium \ + nss \ + freetype \ + harfbuzz \ + ca-certificates \ + ttf-freefont COPY package*.json ./ +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ + PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + # run aws cli without mounting secret, because CI already has AWS credentials FROM build_base AS ci_image diff --git a/docsfera.json b/docsfera.json index c2b9b59..b8d9671 100644 --- a/docsfera.json +++ b/docsfera.json @@ -20,6 +20,14 @@ ], "type": "string" } + }, + { + "name": "origin", + "required": true, + "in": "header", + "schema": { + "type": "string" + } } ], "requestBody": { @@ -4970,6 +4978,14 @@ ], "type": "string" } + }, + { + "name": "type", + "required": true, + "in": "query", + "schema": { + "type": "string" + } } ], "responses": { diff --git a/package-lock.json b/package-lock.json index 558985c..c9d852f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.23", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -30,6 +30,7 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cron-parser": "^4.9.0", + "csv": "^6.3.11", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", "handlebars": "^4.7.8", @@ -44,6 +45,7 @@ "passport-google-oauth20": "^2.0.0", "passport-hubspot-oauth2": "^1.0.3", "passport-mailchimp": "^1.1.0", + "puppeteer": "^24.7.2", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", "rxjs": "^7.5.5", @@ -1398,9 +1400,9 @@ } }, "node_modules/@dadosfera/protospack-v2": { - "version": "0.0.0", - "resolved": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", - "integrity": "sha512-3vMGyEolF/xeEeFSRUpOA8YVrbTZfWXrcMxMSCkmx411Tuuv4SkX9nHcYaXhYAkFUV9vR7vfmBfMFz4yq1n7mw==", + "version": "3.37.0-beta.23", + "resolved": "https://dadosfera-611330257153.d.codeartifact.us-east-1.amazonaws.com/npm/dadosfera-npm/@dadosfera/protospack-v2/-/protospack-v2-3.37.0-beta.23.tgz", + "integrity": "sha512-Dv3rODwbiubHB4u8sI5uJOSm1ETOXzlpaxHNEmzTNG2fLTIJxsaF1GlOBST3x9HTkhJC6hUJ8r77/qQu1FKPpQ==", "license": "ISC", "dependencies": { "@grpc/grpc-js": "^1.9.3", @@ -2946,6 +2948,27 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", "license": "BSD-3-Clause" }, + "node_modules/@puppeteer/browsers": { + "version": "2.10.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.2.tgz", + "integrity": "sha512-i4Ez+s9oRWQbNjtI/3+jxr7OH508mjAKvza0ekPJem0ZtmsYHP3B5dq62+IaBHKaGCOuqJxXzvFLUhJvQ6jtsQ==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.0", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.1", + "tar-fs": "^3.0.8", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@sinonjs/commons": { "version": "1.8.6", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.6.tgz", @@ -3541,6 +3564,12 @@ "node": ">= 6" } }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", @@ -3986,6 +4015,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.62.0.tgz", @@ -4636,6 +4675,18 @@ "safer-buffer": "^2.1.0" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", @@ -4708,6 +4759,12 @@ "form-data": "^4.0.0" } }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, "node_modules/babel-jest": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.5.1.tgz", @@ -4814,6 +4871,78 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/bare-events": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz", + "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.1.3.tgz", + "integrity": "sha512-OeEZYIg+2qepaWLyphaOXHAHKo3xkM8y3BeGAvHdMN8GNWvEAU1Yw6rYpGzu/wDDbKxgEjVeVDpgGhDzaeMpjg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.1.tgz", + "integrity": "sha512-uaIjxokhFidJP+bmmvKSgiMzj2sV5GPHaZVAIktcxcpCyBFFWO+YlikVAdhmUo2vYFvFhOXIAlldqV29L8126g==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.5.tgz", + "integrity": "sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -4855,6 +4984,15 @@ "node": ">= 0.8" } }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -5058,6 +5196,15 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -5244,6 +5391,19 @@ "node": ">=6.0" } }, + "node_modules/chromium-bidi": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-4.1.1.tgz", + "integrity": "sha512-biR7t4vF3YluE6RlMSk9IWk+b9U+WWyzHp+N2pL9vRTk+UXHYRTVp7jTK58ZNzMLBgoLMHY4QyJMbeuw3eKxqg==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/ci-info": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", @@ -5688,6 +5848,48 @@ "dev": true, "license": "MIT" }, + "node_modules/csv": { + "version": "6.3.11", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.11.tgz", + "integrity": "sha512-a8bhT76Q546jOElHcTrkzWY7Py925mfLO/jqquseH61ThOebYwOjLbWHBqdRB4K1VpU36sTyIei6Jwj7QdEZ7g==", + "license": "MIT", + "dependencies": { + "csv-generate": "^4.4.2", + "csv-parse": "^5.6.0", + "csv-stringify": "^6.5.2", + "stream-transform": "^3.3.3" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.4.2.tgz", + "integrity": "sha512-W6nVsf+rz0J3yo9FOjeer7tmzBJKaTTxf7K0uw6GZgRocZYPVpuSWWa5/aoWWrjQZj4/oNIKTYapOM7hiNjVMA==", + "license": "MIT" + }, + "node_modules/csv-parse": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz", + "integrity": "sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==", + "license": "MIT" + }, + "node_modules/csv-stringify": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.2.tgz", + "integrity": "sha512-RFPahj0sXcmUyjrObAK+DOWtMvMIFV328n4qZJhgX3x2RqkQgOTU2mCUmiFR0CzM6AzChlRSUErjiJeEt8BaQA==", + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/data-urls": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-2.0.0.tgz", @@ -5802,6 +6004,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -5840,6 +6056,12 @@ "node": ">=8" } }, + "node_modules/devtools-protocol": { + "version": "0.0.1425554", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1425554.tgz", + "integrity": "sha512-uRfxR6Nlzdzt0ihVIkV+sLztKgs7rgquY/Mhcv1YNCWDh5IZgl5mnn2aeEnW5stYTE0wwiF4RYVz8eMEpV1SEw==", + "license": "BSD-3-Clause" + }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", @@ -6142,6 +6364,15 @@ "node": ">=10.13.0" } }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/error-callsites": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/error-callsites/-/error-callsites-2.0.4.tgz", @@ -6251,7 +6482,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", - "dev": true, "license": "BSD-2-Clause", "dependencies": { "esprima": "^4.0.1", @@ -6273,7 +6503,6 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=4.0" @@ -6283,7 +6512,6 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, "license": "BSD-3-Clause", "optional": true, "engines": { @@ -6494,7 +6722,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, "license": "BSD-2-Clause", "bin": { "esparse": "bin/esparse.js", @@ -6561,7 +6788,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "license": "BSD-2-Clause", "engines": { "node": ">=0.10.0" @@ -6791,6 +7017,41 @@ "node": ">=4" } }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -6804,6 +7065,12 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", @@ -6900,6 +7167,15 @@ "bser": "2.1.1" } }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, "node_modules/fecha": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", @@ -7273,6 +7549,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-uri": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.4.tgz", + "integrity": "sha512-E1b1lFFLvLgak2whF2xDBcOy6NLVGZBqqjJjsIhvopKfWWEi64pLVTWWehV8KlLerZkfNTA95sTe2OdJKm1OzQ==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -7755,6 +8045,25 @@ "node": ">= 0.10" } }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "license": "MIT", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/ipaddr.js": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", @@ -8753,6 +9062,12 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==", + "license": "MIT" + }, "node_modules/jsdom": { "version": "16.7.0", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-16.7.0.tgz", @@ -9442,6 +9757,12 @@ "node": ">=8" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/mixpanel": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/mixpanel/-/mixpanel-0.17.0.tgz", @@ -9553,6 +9874,15 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "license": "MIT" }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/next-line": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/next-line/-/next-line-1.1.0.tgz", @@ -9906,6 +10236,73 @@ "node": ">=6" } }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -10186,6 +10583,12 @@ "resolved": "https://registry.npmjs.org/pause/-/pause-0.0.1.tgz", "integrity": "sha512-KG8UEiEVkR3wGEb4m5yZkVCzigAD+cVEJck2CzYZO37ZGJfctvVptVO192MwrtPhzONn6go8ylnOdMhKqi4nfg==" }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -10403,6 +10806,15 @@ "integrity": "sha512-du4wfLyj4yCZq1VupnVSZmRsPJsNuxoDQFdCFHLaYiEbFBD7QE0a+I4D7hOxrVnh78QE/YipFAj9lXHiXocV+Q==", "license": "MIT" }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -10464,6 +10876,75 @@ "node": ">= 0.10" } }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -10502,6 +10983,91 @@ "node": ">=6" } }, + "node_modules/puppeteer": { + "version": "24.7.2", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.7.2.tgz", + "integrity": "sha512-ifYqoY6wGs0yZeFuFPn8BE9FhuveXkarF+eO18I2e/axdoCh4Qh1AE+qXdJBhdaeoPt6eRNTY4Dih29Jbq8wow==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.10.2", + "chromium-bidi": "4.1.1", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1425554", + "puppeteer-core": "24.7.2", + "typed-query-selector": "^2.12.0" + }, + "bin": { + "puppeteer": "lib/cjs/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core": { + "version": "24.7.2", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.7.2.tgz", + "integrity": "sha512-P9pZyTmJqKODFCnkZgemCpoFA4LbAa8+NumHVQKyP5X9IgdNS1ZnAnIh1sMAwhF8/xEUGf7jt+qmNLlKieFw1Q==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.10.2", + "chromium-bidi": "4.1.1", + "debug": "^4.4.0", + "devtools-protocol": "0.0.1425554", + "typed-query-selector": "^2.12.0", + "ws": "^8.18.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core/node_modules/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/puppeteer/node_modules/cosmiconfig": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", + "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/qs": { "version": "6.11.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", @@ -11179,6 +11745,53 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.4", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.4.tgz", + "integrity": "sha512-D3YaD0aRxR3mEcqnidIs7ReYJFVzWdd6fXJYUM8ixcQcJRGTka/b3saV0KflYhyVJXKhb947GndU35SxYNResQ==", + "license": "MIT", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks-proxy-agent/node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/sonic-boom": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.4.1.tgz", @@ -11286,6 +11899,12 @@ "readable-stream": "^3.0.6" } }, + "node_modules/stream-transform": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.3.tgz", + "integrity": "sha512-dALXrXe+uq4aO5oStdHKlfCM/b3NBdouigvxVPxCdrMRAU6oHh3KNss20VbTPQNQmjAHzZGKGe66vgwegFEIog==", + "license": "MIT" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -11294,6 +11913,19 @@ "node": ">=10.0.0" } }, + "node_modules/streamx": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.0.tgz", + "integrity": "sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -11541,6 +12173,31 @@ "node": ">=6" } }, + "node_modules/tar-fs": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz", + "integrity": "sha512-ZoROL70jptorGAlgAYiLoBLItEKw/fUxg9BSYK/dF/GAGYFJOJJJMvjPAKDJraCXFwadD456FCuvLWgfhMsPwg==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/terminal-link": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/terminal-link/-/terminal-link-2.1.1.tgz", @@ -11650,6 +12307,15 @@ "node": ">=8" } }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-hex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", @@ -12065,6 +12731,12 @@ "node": ">= 0.6" } }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", + "license": "MIT" + }, "node_modules/typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", @@ -12740,6 +13412,16 @@ "node": ">=12" } }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, "node_modules/yn": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", @@ -12762,6 +13444,15 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.24.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.3.tgz", + "integrity": "sha512-HhY1oqzWCQWuUqvBFnsyrtZRhyPeR7SUGv+C4+MsisMuVfSPx8HpwWqH8tRahSlt6M3PiFAcoeFhZAqIXTxoSg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index ae1463a..fd3eb0b 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "@aws-sdk/client-secrets-manager": "^3.414.0", "@dadosfera/dadosfera-logs": "^1.0.0-beta.4", "@dadosfera/protospack": "2.5.3", - "@dadosfera/protospack-v2": "file:../protospack-v2/dadosfera-protospack-v2-0.0.0.tgz", + "@dadosfera/protospack-v2": "3.37.0-beta.23", "@grpc/grpc-js": "^1.9.3", "@grpc/proto-loader": "^0.7.9", "@nestjs/cli": "^9.5.0", @@ -48,6 +48,7 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cron-parser": "^4.9.0", + "csv": "^6.3.11", "dotenv": "^14.3.2", "elastic-apm-node": "^3.50.0", "handlebars": "^4.7.8", @@ -62,6 +63,7 @@ "passport-google-oauth20": "^2.0.0", "passport-hubspot-oauth2": "^1.0.3", "passport-mailchimp": "^1.1.0", + "puppeteer": "^24.7.2", "reflect-metadata": "^0.1.13", "rimraf": "^3.0.2", "rxjs": "^7.5.5", diff --git a/src/modules/catalog/catalog.controller.ts b/src/modules/catalog/catalog.controller.ts index 9f80e35..ab6290a 100644 --- a/src/modules/catalog/catalog.controller.ts +++ b/src/modules/catalog/catalog.controller.ts @@ -346,7 +346,7 @@ export class CatalogController { @Language() language: LanguageEnum, @Param('id') id: string, ): Promise { - const { customer_name, customer_id, user_id, username } = user; + const { customer_name, customer_id, user_id, username, customer_modules } = user; this.logger.info(`/catalog - ON GET DATA DOCS ROUTE`, { user_id, @@ -359,6 +359,7 @@ export class CatalogController { user_id, username, language, + is_mask: customer_modules.some(mod => mod === 'pii') }); const preview = await this.catalogService.getDatasetPreview(id, metadata); @@ -863,15 +864,27 @@ export class CatalogController { @RequireModule( DADOSFERA_MODULES_KEYS.PII ) - async getPiiReporter(@User() user: RequestUser, @Res() res: Response) { + async getPiiReporter(@User() user: RequestUser, @Res() res: Response, @Query('type') type: "html" | "csv") { this.logger.info('GET pii-reporter'); const metadata = PackTheMetadata(user); try { - const reporterHtml = await this.catalogService.getPiiReporter(metadata); + const reporter = await this.catalogService.getPiiReporter(metadata, type); - res.setHeader('Content-Type', 'text/html'); - return res.send(reporterHtml); + if (type === "html") { + res.setHeader('Content-Type', 'text/html'); + } else if (type === "csv") { + res.setHeader('Content-Type', 'text/csv'); + } else if (type === "pdf") { + res.setHeader('Content-Type', 'application/pdf'); + } + + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + const filename = `relatorio-pii-${timestamp}.${type}`; + + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + + return res.send(reporter); } catch (error) { this.logger.error(error); } diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts index ac7f1d6..68036c3 100644 --- a/src/modules/catalog/catalog.service.ts +++ b/src/modules/catalog/catalog.service.ts @@ -34,8 +34,12 @@ import { GetNimbusDashboardsRequest, GetRlsRulesRequest, } from '@dadosfera/protospack-v2/dist/lib/Catalog/interfaces/messages'; -import { readFileSync } from 'fs'; +import { writeFileSync } from 'fs'; import path from 'path'; +import { HtmlParser } from 'src/utils/FileParser/html-parser'; +import { PiiDto } from './dtos/pii.dto'; +import { CsvParser } from 'src/utils/FileParser/csv-parser'; +import { PDFParser } from 'src/utils/FileParser/pdf-parser'; class CatalogService implements OnModuleInit { catalogReadService: ReadService.CatalogReadServices; @@ -82,49 +86,47 @@ class CatalogService implements OnModuleInit { )}.dadosfera.ai`; } - async getPiiReporter(metadata: Metadata) { - const { - data - } = await lastValueFrom( - this.catalogWriteService.GetPiiReporter({}, metadata) - ) + async getPiiReporter(metadata: Metadata, type: "html"| "csv") { + try { + const { + data + } = await lastValueFrom( + this.catalogWriteService.GetPiiReporter({}, metadata) + ) + this.logger.info("Finish grpc call") + + // 3. Ler o template HTML + const templatePath = path.join( + __dirname, + '../../../src/template/pii-reporter.html', + ); - const result = data.map((props) => ({ - ...props, - pii_rules: this.formatPiiRules(JSON.parse(props.pii_rules)), - })); + this.logger.info("templatePath: " + templatePath); - // 3. Ler o template HTML - const templatePath = path.join( - __dirname, - '../../../src/template/pii-reporter.html', - ); - const templateSource = readFileSync(templatePath, 'utf8'); - - // 4. Compilar o template - const template = Handlebars.compile(templateSource); - - // 5. Aplicar os dados ao template - return template({ - dados: result, - dataGeracao: new Date().toLocaleString('pt-BR'), - }); - } - - private formatPiiRules( - data: Record< - string, - { - avg_score: number; - max_score: number; + if (type === "html") { + const htmlParser = new HtmlParser(templatePath); + return htmlParser.parse(data); + } else if (type === "csv") { + const csvParser = new CsvParser(); + return await csvParser.parse(data) + } else if (type === "pdf") { + const htmlParser = new HtmlParser(templatePath); + const pdfParser = new PDFParser(); + const pdf = await pdfParser.parse(htmlParser.parse(data)); + const pdfPath = path.join( + __dirname, + '../../../src/template/teste.pdf', + ); + writeFileSync(pdfPath, pdf); + return pdf } - >, - ) { - return Object.keys(data) - .filter((key) => { - return data[key].max_score > 0.7; - }) - .join(', '); + + throw new Error("Unsupported Type: " + type) + + } catch (error) { + this.logger.error(error); + } + } async createDataAsset(data: Messages.CreateDataAssetRequest, metadata) { diff --git a/src/modules/catalog/dtos/pii.dto.ts b/src/modules/catalog/dtos/pii.dto.ts new file mode 100644 index 0000000..c456b2b --- /dev/null +++ b/src/modules/catalog/dtos/pii.dto.ts @@ -0,0 +1,8 @@ +export class PiiDto { + database_name: string; + table_schema: string; + table_name: string; + column_name: string; + data_type: string; + pii_rules: string; +} diff --git a/src/template/pii-reporter.html b/src/template/pii-reporter.html index 637332a..d4b4505 100644 --- a/src/template/pii-reporter.html +++ b/src/template/pii-reporter.html @@ -6,6 +6,10 @@ Dadosfera Relatório de PII