diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63d095f..977c0fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,29 +6,18 @@ on: jobs: test: - runs-on: [self-hosted, prd] + runs-on: [self-hosted, prd] + env: + APP_NAME: ${{ github.event.repository.name }} steps: - name: Checkout uses: actions/checkout@v4 - - name: Configure AWS - uses: aws-actions/configure-aws-credentials@v4 - id: aws - with: - aws-region: us-east-1 - - name: Build - env: - APP_NAME: ${{ github.event.repository.name }} - run: | - docker build -t $APP_NAME --target test . + run: docker build -t ${APP_NAME}_teste --target test . - name: Run Test - env: - ENV: test - APP_NAME: ${{ github.event.repository.name }} - run: | - docker run --rm --entrypoint="npm" $APP_NAME run test + run: docker run ${APP_NAME}_teste - name: Remove Docker's Trash if: always() diff --git a/.nvmrc b/.nvmrc index bcaa337..aacb518 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -18.17 \ No newline at end of file +18.17 diff --git a/Dockerfile b/Dockerfile index ddab8ad..e407e07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,42 @@ -# install all dependencies -FROM node:18.17-alpine as packages +FROM node:18.17-alpine AS base_image + +FROM base_image AS build_base WORKDIR /app -COPY package.json package-lock.json ./ -RUN apk add --no-cache aws-cli \ - && aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 \ - && npm install +RUN apk update +# needed packages to build dependencies from source +RUN apk add --no-cache aws-cli +COPY package*.json ./ + + +# run aws cli without mounting secret, because CI already has AWS credentials +FROM build_base AS ci_image +RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 +RUN npm ci +COPY . . + # unit test specific build -FROM node:18.17-alpine as test -WORKDIR /app -COPY --from=packages /app/node_modules ./node_modules -COPY . ./ +FROM ci_image AS test +ENTRYPOINT ["npm", "run", "test"] -FROM node:18.17-alpine -WORKDIR /app -COPY . /app/ -COPY --from=packages /app/node_modules ./node_modules + +# dev build +FROM build_base AS dev +RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \ + aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 +# flag --build-from-source is required to force-build sqlite3 +RUN npm ci +COPY . . +ENTRYPOINT npm run start:dev + + +FROM ci_image AS prod_build RUN npm run build -EXPOSE 3333 -ENTRYPOINT npm run start + + +FROM base_image +WORKDIR /app +COPY --from=prod_build /app/dist ./dist +COPY --from=prod_build /app/node_modules ./node_modules +COPY --from=prod_build /app/package*.json ./ +ENTRYPOINT npm run start:prod diff --git a/build.docker-compose.dockerhub.yml b/build.docker-compose.dockerhub.yml index 7c21c52..31dd969 100644 --- a/build.docker-compose.dockerhub.yml +++ b/build.docker-compose.dockerhub.yml @@ -1,5 +1,4 @@ -version: "3.8" services: maestro: build: . - image: dadosfera/maestro_${ENV}:${IMAGE_TAG} \ No newline at end of file + image: dadosfera/maestro_${ENV}:${IMAGE_TAG} diff --git a/build.docker-compose.yml b/build.docker-compose.yml index 62b9307..7dddc0e 100644 --- a/build.docker-compose.yml +++ b/build.docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.8" services: maestro: build: . diff --git a/docker-compose.dockerhub.yml b/docker-compose.dockerhub.yml index 6747892..bac26b8 100644 --- a/docker-compose.dockerhub.yml +++ b/docker-compose.dockerhub.yml @@ -1,4 +1,3 @@ -version: "3.8" services: maestro: image: dadosfera/maestro_${ENV}:${IMAGE_TAG} diff --git a/docker-compose.yml b/docker-compose.yml index 3b79ae6..3489bfd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.8" services: maestro: image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG} diff --git a/docsfera.json b/docsfera.json index 1df2238..0922da4 100644 --- a/docsfera.json +++ b/docsfera.json @@ -5447,7 +5447,7 @@ } }, "info": { - "title": "Maestro - feat/rls", + "title": "Maestro", "description": "This is the Maestro API", "version": "1.0.0", "contact": {} @@ -8238,4 +8238,4 @@ } } } -} \ No newline at end of file +} diff --git a/package-lock.json b/package-lock.json index 61e212a..61c7a9a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,6 @@ "": { "name": "maestro", "version": "0.0.0", - "hasInstallScript": true, "license": "UNLICENSED", "dependencies": { "@aws-sdk/client-secrets-manager": "^3.414.0", @@ -77,7 +76,7 @@ "typescript": "^4.9.5" }, "engines": { - "node": "^18.17.0" + "node": "18.17" } }, "node_modules/@ampproject/remapping": { diff --git a/package.json b/package.json index b750615..fbe7eea 100644 --- a/package.json +++ b/package.json @@ -6,10 +6,10 @@ "private": true, "license": "UNLICENSED", "engines": { - "node": "^18.17.0" + "node": "18.17" }, "scripts": { - "preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1", + "co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1", "proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact", "prebuild": "rimraf dist", "build": "nest build", @@ -20,7 +20,7 @@ "start:prod": "node dist/main", "docs": "export KILL_AFTER_START=1 && nest start && export INTERNAL_SWAGGER=true && nest start", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", - "test": "jest", + "test": "jest --detectOpenHandles --forceExit", "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", diff --git a/src/modules/catalog/catalog-client.ts b/src/modules/catalog/catalog-client.ts index 5d6208c..0307104 100644 --- a/src/modules/catalog/catalog-client.ts +++ b/src/modules/catalog/catalog-client.ts @@ -6,7 +6,9 @@ import { import { credentials } from '@grpc/grpc-js'; import { Catalog } from '@dadosfera/protospack-v2'; -const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.PIFACTORY_URL.startsWith('pi-factory:') || + process.env.PIFACTORY_URL?.includes('0.0.0.0'); export class CatalogClientConfiguration { public name = 'CatalogClientConfiguration'; diff --git a/src/modules/connection-test/connection-test-client.config.ts b/src/modules/connection-test/connection-test-client.config.ts index 6c042c9..e55ab24 100644 --- a/src/modules/connection-test/connection-test-client.config.ts +++ b/src/modules/connection-test/connection-test-client.config.ts @@ -6,7 +6,9 @@ import { } from '@nestjs/microservices'; import { ConnectionTest } from '@dadosfera/protospack-v2'; -const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.INFACTORY_URL.startsWith('in-factory:') || + process.env.INFACTORY_URL.includes('0.0.0.0'); export class ConnectionTestClientConfiguration { private config: GrpcOptions = { diff --git a/src/modules/connection/client.config.ts b/src/modules/connection/client.config.ts index 938ef37..e9b7c78 100644 --- a/src/modules/connection/client.config.ts +++ b/src/modules/connection/client.config.ts @@ -6,7 +6,9 @@ import { } from '@nestjs/microservices'; import { ConnectionManager } from '@dadosfera/protospack-v2'; -const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.INFACTORY_URL.startsWith('in-factory:') || + process.env.INFACTORY_URL.includes('0.0.0.0'); export class ConnectionClientConfiguration { public name = 'ConnectionClientConfiguration'; diff --git a/src/modules/connector/client.config.ts b/src/modules/connector/client.config.ts index fdbd5a1..c9b82ce 100644 --- a/src/modules/connector/client.config.ts +++ b/src/modules/connector/client.config.ts @@ -6,7 +6,9 @@ import { } from '@nestjs/microservices'; import { ConnectorManager } from '@dadosfera/protospack-v2'; -const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.INFACTORY_URL.startsWith('in-factory:') || + process.env.INFACTORY_URL.includes('0.0.0.0'); export class ConnectorClientConfiguration { public name = 'ConnectorClientConfiguration'; diff --git a/src/modules/duc/client.config.ts b/src/modules/duc/client.config.ts index 1bcd08e..c5c684e 100644 --- a/src/modules/duc/client.config.ts +++ b/src/modules/duc/client.config.ts @@ -9,7 +9,9 @@ import { ProtoPaths, } from '@dadosfera/protospack-v2/dist/lib/Duc'; -const isLocalConnection = !!process.env.DUC_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.DUC_URL.startsWith('duc:') || + !!process.env.DUC_URL?.includes('0.0.0.0'); export class DucClient { public name = 'DucClient'; diff --git a/src/modules/inputs/inputs-client.config.ts b/src/modules/inputs/inputs-client.config.ts index b52009f..23bd919 100644 --- a/src/modules/inputs/inputs-client.config.ts +++ b/src/modules/inputs/inputs-client.config.ts @@ -6,7 +6,9 @@ import { type GrpcOptions, } from '@nestjs/microservices'; -const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.INFACTORY_URL.startsWith('in-factory:') || + process.env.INFACTORY_URL.includes('0.0.0.0'); export class InputsGrpcClient { public readonly name = 'InputsGrpcClient'; private config: GrpcOptions = { diff --git a/src/modules/network-config/network-config-client.config.ts b/src/modules/network-config/network-config-client.config.ts index a5227f8..284f4c2 100644 --- a/src/modules/network-config/network-config-client.config.ts +++ b/src/modules/network-config/network-config-client.config.ts @@ -6,7 +6,9 @@ import { } from '@nestjs/microservices'; import { NetworkConfig } from '@dadosfera/protospack-v2'; -const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.INFACTORY_URL.startsWith('in-factory:') || + process.env.INFACTORY_URL.includes('0.0.0.0'); export class NetworkConfigGrpcClient { public name = 'NetworkConfigGrpcClient'; diff --git a/src/modules/pipelines/pipelines-client.ts b/src/modules/pipelines/pipelines-client.ts index 1c66489..3d4c5e5 100644 --- a/src/modules/pipelines/pipelines-client.ts +++ b/src/modules/pipelines/pipelines-client.ts @@ -6,7 +6,9 @@ import { import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack'; import { credentials } from '@grpc/grpc-js'; -const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.PIFACTORY_URL.startsWith('pi-factory:') || + process.env.PIFACTORY_URL?.includes('0.0.0.0'); export class PipelinesClientConfiguration { public name = 'PipelinesClientConfiguration'; diff --git a/src/modules/pipelinesV2/pipelines-client.ts b/src/modules/pipelinesV2/pipelines-client.ts index 118e890..e998624 100644 --- a/src/modules/pipelinesV2/pipelines-client.ts +++ b/src/modules/pipelinesV2/pipelines-client.ts @@ -9,7 +9,9 @@ import { } from '@dadosfera/protospack-v2/dist/lib/PipelineV2'; import { credentials } from '@grpc/grpc-js'; -const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.PIFACTORY_URL.startsWith('pi-factory:') || + !!process.env.PIFACTORY_URL?.includes('0.0.0.0'); export class PipelinesClientConfiguration { public name = 'PipelinesClientConfiguration'; diff --git a/src/modules/transformations/transformations-client.ts b/src/modules/transformations/transformations-client.ts index 9b9a14d..252bb84 100644 --- a/src/modules/transformations/transformations-client.ts +++ b/src/modules/transformations/transformations-client.ts @@ -6,7 +6,9 @@ import { import { credentials } from '@grpc/grpc-js'; import { Transformation } from '@dadosfera/protospack-v2'; -const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0'); +const isLocalConnection = + process.env.INFACTORY_URL.startsWith('in-factory:') || + process.env.INFACTORY_URL.includes('0.0.0.0'); export class TransformationsClientConfiguration { public name = 'TransformationsClientConfiguration';