Compare commits

..
9 changed files with 156 additions and 73 deletions
+11 -34
View File
@@ -142,11 +142,17 @@ jobs:
docker system prune --volumes -a -f
docker system df
k8s-setup:
needs: [extract_environment]
uses: ./.github/workflows/k8s-setup.yml
with:
cloud: azure
environment: ${{ needs.extract_environment.outputs.environment }}
secrets: inherit
helmfile-deploy:
needs: [extract_environment, semantic_release, build_ecr_image]
env:
HOME: /home/runner
runs-on: [self-hosted, "prd-oracle"]
needs: [extract_environment, semantic_release, build_ecr_image, k8s-setup]
runs-on: [self-hosted, "prd-azure"]
environment: ${{ needs.extract_environment.outputs.environment }}
steps:
@@ -158,18 +164,6 @@ jobs:
with:
version: 'v3.9.0'
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Set up Python
uses: actions/setup-python@v4
with:
@@ -179,29 +173,12 @@ jobs:
run: |
wget https://github.com/helmfile/helmfile/releases/download/v0.148.0/helmfile_0.148.0_linux_amd64.tar.gz
tar -xzf helmfile_0.148.0_linux_amd64.tar.gz
sudo mv helmfile /usr/local/bin/
mv helmfile /usr/local/bin/
helmfile --version
- name: Install Helm Diff Plugin
run: helm plugin install https://github.com/databus23/helm-diff || true
- name: Authenticate with OKE cluster
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Run Helmfile Apply
env:
ENV: ${{ needs.extract_environment.outputs.environment }}
+77
View File
@@ -0,0 +1,77 @@
name : K8s Setup kube config to deploy
on:
workflow_call:
inputs:
cloud:
description: "Cloud provider for the deployment"
required: true
default: "azure"
type: string
environment:
description: "Deployment environment"
required: true
default: "prd"
type: string
jobs:
azure:
if: inputs.cloud == 'azure'
runs-on: [self-hosted, "prd-azure"]
steps:
- name: Install Azure ClI
run: |
curl -sL https://aka.ms/InstallAzureCLIDeb | bash
- uses: azure/login@v2
with:
creds: '{"clientId":"${{ secrets.ARM_CLIENT_ID }}","clientSecret":"${{ secrets.ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.ARM_TENANT_ID }}"}'
- name: Authenticate with cluster
env:
CLUSTER_NAME: platform-${{ inputs.environment }}
run: az aks get-credentials --resource-group dadosfera-prd --name ${CLUSTER_NAME} --overwrite-existing
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
oci:
if: inputs.cloud == 'oci'
runs-on: [self-hosted, "prd-oracle"]
steps:
- name: Install OCI CLI
run: |
bash -c "$(curl -L https://raw.githubusercontent.com/oracle/oci-cli/master/scripts/install/install.sh)" -- --accept-all-defaults
echo "$HOME/bin" >> $GITHUB_PATH
- name: Configure OCI CLI
run: |
mkdir -p ~/.oci || true
echo "${{ secrets.OCI_CONFIG }}" > ~/.oci/config
echo "${{ secrets.OCI_PRIVATE_KEY }}" > ~/.oci/oci_api_key.pem
chmod 600 ~/.oci/oci_api_key.pem
- name: Authenticate with OKE cluster
env:
ENV: ${{ inputs.environment }}
STG_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaagh3jvln52a3ebm3dodx6emmhv5bmfs7i7sv2k4zkbcbrzcl6v37q"
PRD_CLUSTER_ID: "ocid1.cluster.oc1.sa-saopaulo-1.aaaaaaaanf3vptl6hc2tzd4enfd2hfpsht3wikxww5xejc3l7cwfm6l3sndq"
HOME: /home/runner
run: |
if [ "$ENV" = "stg" ]; then
CLUSTER_ID=$STG_CLUSTER_ID
elif [ "$ENV" = "prd" ]; then
CLUSTER_ID=$PRD_CLUSTER_ID
else
echo "Unknown environment: $ENV"
exit 1
fi
oci ce cluster create-kubeconfig --cluster-id ${CLUSTER_ID} --file $HOME/.kube/config --region sa-saopaulo-1 --token-version 2.0.0 --kube-endpoint PRIVATE_ENDPOINT
- name: Setup kubectl
uses: azure/setup-kubectl@v1
with:
version: 'v1.30.1'
+1 -1
View File
@@ -60,7 +60,7 @@ affinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: name
- key: application
operator: In
values:
- general
+1 -1
View File
@@ -615,7 +615,7 @@ export const DADOSFERA_MODULES_KEYS = {
ACCESS_DASHBOARD: 'access-dashboard',
DANGER_ZONE: 'danger-zone',
PII: 'pii',
PUBLIC_ASSIGN_EMBED: 'public-assign-embed',
EMBED: 'embedded-analytics',
}
export const DADOSFERA_MODULES: Array<DadosferaModule> = [
+2 -2
View File
@@ -15,7 +15,7 @@ export class AssignController {
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.PUBLIC_ASSIGN_EMBED)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED)
create(@Body() createAssignDto: CreateAssignDto, @User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return this.assignService.create(createAssignDto, metadata);
@@ -25,7 +25,7 @@ export class AssignController {
@RequireSomePermission(
PERMISSIONS_GROUPS.USERS.permissions.ADMIN
)
@RequireModule(DADOSFERA_MODULES_KEYS.PUBLIC_ASSIGN_EMBED)
@RequireModule(DADOSFERA_MODULES_KEYS.EMBED)
async get(@User() user: RequestUser) {
const metadata = PackTheMetadata(user);
return await this.assignService.get(metadata);
+1 -1
View File
@@ -33,6 +33,6 @@ export class AssignService implements OnModuleInit {
}
async get(metadata: Metadata) {
return await lastValueFrom(this.ducService.GetAssignPublicKey(metadata))
return await lastValueFrom(this.ducService.GetAssignPublicKey({}, metadata))
}
}
@@ -41,23 +41,22 @@ export class ShareController {
@Get('/:id')
async getShareDataAsset(
@Param('id') id: string,
@User() user: RequestUser,
@Req() request: Request
) {
this.logger.info(`GET //:id`);
return await this.catalogShareService.getOneDataAssetPublic(id, user, request);
return await this.catalogShareService.getOneDataAssetPublic(id, request);
}
@Get('/:id/columns-metadata')
async getShareDataAssetColumnsMetadata(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser
@Req() request: Request
): Promise<IColumnsMetadataResponse> {
this.logger.info(`GET /:id/columns-metadata`);
const columns_metadata =
await this.catalogShareService.getDatasetColumnsMetadata(id, user);
await this.catalogShareService.getDatasetColumnsMetadata(id, request);
return { columns_metadata };
@@ -67,10 +66,10 @@ export class ShareController {
async getShareDataAssetPreview(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser
@Req() request: Request
): Promise<IPreviewResponse> {
this.logger.info(`GET /:id/preview`);
const preview = await this.catalogShareService.getDatasetPreview(id, user);
const preview = await this.catalogShareService.getDatasetPreview(id, request);
return { preview };
}
@@ -79,10 +78,10 @@ export class ShareController {
async getShareDataAssetDocs(
@Language() language: LanguageEnum,
@Param('id') id: string,
@User() user: RequestUser
@Req() request: Request
): Promise<IDocsResponse> {
this.logger.info(`GET /:id/docs`);
const docs = await this.catalogShareService.getDataDocs(id, user);
const docs = await this.catalogShareService.getDataDocs(id, request);
return { docs };
}
+3 -1
View File
@@ -9,6 +9,7 @@ import { ShareController } from "./share.controller";
import DadosferaLogger from "@dadosfera/dadosfera-logs";
import { ShareService } from "./share.service";
import { MixpanelModule } from "src/modules/mixpanel/mixpanel.module";
import { AuthModule } from "src/modules/auth/auth.module";
const client = new CatalogClientConfiguration();
@@ -19,7 +20,8 @@ const client = new CatalogClientConfiguration();
RolesModule,
CustomersModule,
ShareMetadataModule,
MixpanelModule
MixpanelModule,
AuthModule
],
controllers: [ShareController],
providers: [ShareService, DadosferaLogger],
+53 -25
View File
@@ -22,6 +22,9 @@ import { ShareMetadataService } from 'src/modules/share-metadata/share-metadata.
import { isJWT } from 'class-validator';
import { MixpanelService } from 'src/modules/mixpanel/mixpanel.service';
import { Request } from 'express';
import jwt from 'jsonwebtoken';
import { AuthClientService } from 'src/modules/auth/auth.service';
export class ShareService implements OnModuleInit {
catalogReadService: ReadService.CatalogReadServices;
@@ -36,6 +39,7 @@ export class ShareService implements OnModuleInit {
private readonly roleService: RolesService,
private readonly shareMetadataService: ShareMetadataService,
private readonly mixpanelService: MixpanelService,
private authClient: AuthClientService,
) {
this.logger = dadosferaLogger.logger;
}
@@ -47,8 +51,8 @@ export class ShareService implements OnModuleInit {
);
}
async getDatasetColumnsMetadata(id: string, user: RequestUser) {
const shareMetadata = await this.getShareMetadata(id, user);
async getDatasetColumnsMetadata(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
@@ -63,8 +67,8 @@ export class ShareService implements OnModuleInit {
return result;
}
async getDatasetPreview(id: string, user: RequestUser) {
const shareMetadata = await this.getShareMetadata(id, user);
async getDatasetPreview(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
@@ -79,14 +83,14 @@ export class ShareService implements OnModuleInit {
return result;
}
async getOneDataAssetPublic(id: string, user: RequestUser, request: Request) {
async getOneDataAssetPublic(id: string, request: Request) {
this.logger.info("getOneDataAssetPublic: " + JSON.stringify({
id,
userId: user?.user_id,
customerId: user?.customer_id
id
}))
try {
const shareMetadata = await this.getShareMetadata(id, user);
const user = await this.getUserFromRequest(request);
const shareMetadata = await this.getShareMetadata(id, request);
const mixpanelTracker = {
asset: shareMetadata.assetId,
@@ -148,8 +152,8 @@ export class ShareService implements OnModuleInit {
return { data_asset: asset[0] };
}
async getDataDocs(id: string, user: RequestUser) {
const shareMetadata = await this.getShareMetadata(id, user);
async getDataDocs(id: string, request: Request) {
const shareMetadata = await this.getShareMetadata(id, request);
const metadata = PackTheMetadata({
customer_id: shareMetadata.customerId,
customer_name: shareMetadata.customerName,
@@ -197,21 +201,8 @@ export class ShareService implements OnModuleInit {
});
}
// private async validateShareAssign(token: string) {
// const tokenDecoded = jwt.decode(token, {
// complete: true,
// });
// jwt.verify(token, this.pemValue, {
// algorithms: ['RS256'],
// });
// const shareId = (tokenDecoded.payload as JwtPayload).sub;
// const metadata = PackTheMetadata({});
// return await this.shareMetadataService.get(shareId, metadata);
// }
private async getShareMetadata(id: string, user: RequestUser) {
private async getShareMetadata(id: string, request: Request) {
const metadata = PackTheMetadata({});
this.logger.info('GET share metadata')
const info = await this.shareMetadataService.get(id, metadata);
@@ -219,6 +210,8 @@ export class ShareService implements OnModuleInit {
return info;
}
const user = await this.getUserFromRequest(request);
if (info.type === 'private') {
if (!user) {
throw new ForbiddenException(
@@ -244,4 +237,39 @@ export class ShareService implements OnModuleInit {
}
return info;
}
private async getUserFromRequest(request: Request): Promise<RequestUser | null> {
const accessToken = request.get('Authorization');
if (accessToken) {
const accessTokenDecoded: any = jwt.decode(accessToken, {
complete: true,
});
const { kid } = accessTokenDecoded.header;
const { keys } = await this.authClient.getPublicKeys();
const pemValue = keys.find((key) => key.kid === kid);
if (!pemValue) {
return null;
}
jwt.verify(accessToken, pemValue.pem);
const accessTokenPayload = accessTokenDecoded.payload;
return {
user_id: accessTokenPayload.user_id,
username: accessTokenPayload.username,
permissions: accessTokenPayload.permissions,
customer_id: accessTokenPayload.customer_id,
customer_name: accessTokenPayload.customer_name,
customer_tier: accessTokenPayload.customer_tier,
customer_modules: accessTokenPayload.customer_modules,
access_token: accessToken,
};
}
return null;
}
}