FEAT: dev docker

.github/workflows/test.yml

@@ -6,29 +6,18 @@ on:
 
 jobs:
   test:
-    runs-on: [self-hosted, prd] 
+    runs-on: [self-hosted, prd]
+    env:
+      APP_NAME: ${{ github.event.repository.name }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Configure AWS
-        uses: aws-actions/configure-aws-credentials@v4
-        id: aws
-        with:
-          aws-region: us-east-1
-
       - name: Build
-        env:
-          APP_NAME: ${{ github.event.repository.name }}
-        run: |
-          docker build -t $APP_NAME --target test .
+        run: docker build -t ${APP_NAME}_teste --target test .
 
       - name: Run Test
-        env:
-          ENV: test
-          APP_NAME: ${{ github.event.repository.name }}
-        run: |
-          docker run --rm --entrypoint="npm" $APP_NAME run test
+        run: docker run ${APP_NAME}_teste
 
       - name: Remove Docker's Trash
         if: always()

.nvmrc

@@ -1 +1 @@
-18.17
+18.17

Dockerfile

@@ -1,21 +1,42 @@
-# install all dependencies
-FROM node:18.17-alpine as packages
+FROM node:18.17-alpine AS base_image
+
+FROM base_image AS build_base
 WORKDIR /app
-COPY package.json package-lock.json ./
-RUN apk add --no-cache aws-cli \
- && aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1 \
- && npm install
+RUN apk update
+# needed packages to build dependencies from source
+RUN apk add --no-cache aws-cli
+COPY package*.json ./
+
+
+# run aws cli without mounting secret, because CI already has AWS credentials
+FROM build_base AS ci_image
+RUN aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
+RUN npm ci
+COPY . .
+
 
 # unit test specific build
-FROM node:18.17-alpine as test
-WORKDIR /app
-COPY --from=packages /app/node_modules ./node_modules
-COPY . ./
+FROM ci_image AS test
+ENTRYPOINT ["npm", "run", "test"]
 
-FROM node:18.17-alpine
-WORKDIR /app
-COPY . /app/
-COPY --from=packages /app/node_modules ./node_modules
+
+# dev build
+FROM build_base AS dev
+RUN --mount=type=secret,id=aws,target=/root/.aws/credentials \
+  aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1
+# flag --build-from-source is required to force-build sqlite3
+RUN npm ci
+COPY . .
+ENTRYPOINT npm run start:dev
+
+
+FROM ci_image AS prod_build
 RUN npm run build
-EXPOSE 3333
-ENTRYPOINT npm run start
+
+
+FROM base_image
+WORKDIR /app
+COPY --from=prod_build /app/dist ./dist
+COPY --from=prod_build /app/node_modules ./node_modules
+COPY --from=prod_build /app/package*.json ./
+ENTRYPOINT npm run start:prod

build.docker-compose.dockerhub.yml

@@ -1,5 +1,4 @@
-version: "3.8"
 services:
   maestro:
     build: .
-    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}
+    image: dadosfera/maestro_${ENV}:${IMAGE_TAG}

build.docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   maestro:
     build: .

docker-compose.dockerhub.yml

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   maestro:
     image: dadosfera/maestro_${ENV}:${IMAGE_TAG}

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   maestro:
     image: ${ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/microservices/maestro_${ENV}:${IMAGE_TAG}

docsfera.json

@@ -5447,7 +5447,7 @@
       }
     },
     "info": {
-      "title": "Maestro - feat/rls",
+      "title": "Maestro",
       "description": "This is the Maestro API",
       "version": "1.0.0",
       "contact": {}
@@ -8238,4 +8238,4 @@
       }
     }
   }
-}
+}

package-lock.json

@@ -7,7 +7,6 @@
     "": {
       "name": "maestro",
       "version": "0.0.0",
-      "hasInstallScript": true,
       "license": "UNLICENSED",
       "dependencies": {
         "@aws-sdk/client-secrets-manager": "^3.414.0",
@@ -77,7 +76,7 @@
         "typescript": "^4.9.5"
       },
       "engines": {
-        "node": "^18.17.0"
+        "node": "18.17"
       }
     },
     "node_modules/@ampproject/remapping": {

package.json

@@ -6,10 +6,10 @@
   "private": true,
   "license": "UNLICENSED",
   "engines": {
-    "node": "^18.17.0"
+    "node": "18.17"
   },
   "scripts": {
-    "preinstall": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
+    "co:login": "aws codeartifact login --tool npm --namespace @dadosfera --repository dadosfera-npm --domain dadosfera --domain-owner 611330257153 --region us-east-1",
     "proto-update": "npm i @dadosfera/protospack-v2@latest --save-exact",
     "prebuild": "rimraf dist",
     "build": "nest build",
@@ -20,7 +20,7 @@
     "start:prod": "node dist/main",
     "docs": "export KILL_AFTER_START=1 && nest start && export INTERNAL_SWAGGER=true && nest start",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
-    "test": "jest",
+    "test": "jest --detectOpenHandles --forceExit",
     "test:watch": "jest --watch",
     "test:cov": "jest --coverage",
     "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",

src/modules/catalog/catalog-client.ts

@@ -6,7 +6,9 @@ import {
 import { credentials } from '@grpc/grpc-js';
 import { Catalog } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
+  process.env.PIFACTORY_URL?.includes('0.0.0.0');
 
 export class CatalogClientConfiguration {
   public name = 'CatalogClientConfiguration';

src/modules/connection-test/connection-test-client.config.ts

@@ -6,7 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { ConnectionTest } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class ConnectionTestClientConfiguration {
   private config: GrpcOptions = {

src/modules/connection/client.config.ts

@@ -6,7 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { ConnectionManager } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class ConnectionClientConfiguration {
   public name = 'ConnectionClientConfiguration';

src/modules/connector/client.config.ts

@@ -6,7 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { ConnectorManager } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class ConnectorClientConfiguration {
   public name = 'ConnectorClientConfiguration';

src/modules/duc/client.config.ts

@@ -9,7 +9,9 @@ import {
   ProtoPaths,
 } from '@dadosfera/protospack-v2/dist/lib/Duc';
 
-const isLocalConnection = !!process.env.DUC_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.DUC_URL.startsWith('duc:') ||
+  !!process.env.DUC_URL?.includes('0.0.0.0');
 
 export class DucClient {
   public name = 'DucClient';

src/modules/inputs/inputs-client.config.ts

@@ -6,7 +6,9 @@ import {
   type GrpcOptions,
 } from '@nestjs/microservices';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 export class InputsGrpcClient {
   public readonly name = 'InputsGrpcClient';
   private config: GrpcOptions = {

src/modules/network-config/network-config-client.config.ts

@@ -6,7 +6,9 @@ import {
 } from '@nestjs/microservices';
 import { NetworkConfig } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class NetworkConfigGrpcClient {
   public name = 'NetworkConfigGrpcClient';

src/modules/pipelines/pipelines-client.ts

@@ -6,7 +6,9 @@ import {
 import { PipelinePackages, PipelineProtoFilePath } from '@dadosfera/protospack';
 import { credentials } from '@grpc/grpc-js';
 
-const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
+  process.env.PIFACTORY_URL?.includes('0.0.0.0');
 
 export class PipelinesClientConfiguration {
   public name = 'PipelinesClientConfiguration';

src/modules/pipelinesV2/pipelines-client.ts

@@ -9,7 +9,9 @@ import {
 } from '@dadosfera/protospack-v2/dist/lib/PipelineV2';
 import { credentials } from '@grpc/grpc-js';
 
-const isLocalConnection = !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.PIFACTORY_URL.startsWith('pi-factory:') ||
+  !!process.env.PIFACTORY_URL?.includes('0.0.0.0');
 
 export class PipelinesClientConfiguration {
   public name = 'PipelinesClientConfiguration';

src/modules/transformations/transformations-client.ts

@@ -6,7 +6,9 @@ import {
 import { credentials } from '@grpc/grpc-js';
 import { Transformation } from '@dadosfera/protospack-v2';
 
-const isLocalConnection = !!process.env.INFACTORY_URL?.includes('0.0.0.0');
+const isLocalConnection =
+  process.env.INFACTORY_URL.startsWith('in-factory:') ||
+  process.env.INFACTORY_URL.includes('0.0.0.0');
 
 export class TransformationsClientConfiguration {
   public name = 'TransformationsClientConfiguration';