# Configuración MIG para Kubernetes node con NVIDIA A100
# Paso 1: Habilitar MIG en el node (ssh al node, requiere reboot)

# Habilitar MIG mode
sudo nvidia-smi -mig 1

# Crear MIG instances (ejemplo: 3x 1g.5gb profiles + 1x 3g.20gb profile)
sudo nvidia-smi mig -cgi 1g.5gb -C # Crea compute instance
sudo nvidia-smi mig -cgi 1g.5gb -C
sudo nvidia-smi mig -cgi 1g.5gb -C
sudo nvidia-smi mig -cgi 3g.20gb -C

# Verificar MIG instances creadas
sudo nvidia-smi mig -lgi

# Output esperado:
# +----+------+-------+-----------+
# | ID | Name | Size  | Profile   |
# +====+======+=======+===========+
# |  0 | MIG  | 5 GB  | 1g.5gb    |
# |  1 | MIG  | 5 GB  | 1g.5gb    |
# |  2 | MIG  | 5 GB  | 1g.5gb    |
# |  3 | MIG  | 20 GB | 3g.20gb   |
# +----+------+-------+-----------+

---
# Paso 2: Configurar NVIDIA device plugin para exponer MIG instances
apiVersion: v1
kind: ConfigMap
metadata:
  name: nvidia-device-plugin-config
  namespace: kube-system
data:
  config.yaml: |
    version: v1
    sharing:
      mig:
        strategy: mixed # Expone MIG instances + whole GPUs
    migStrategy: mixed
    failOnInitError: true

---
# Paso 3: DaemonSet NVIDIA device plugin (detecta MIG instances)
# Este DaemonSet corre en TODOS los GPU nodes y registra resources en Kubernetes
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: nvidia-device-plugin-daemonset
  namespace: kube-system
spec:
  selector:
    matchLabels:
      name: nvidia-device-plugin-ds
  template:
    metadata:
      labels:
        name: nvidia-device-plugin-ds
    spec:
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
      # Prioridad alta para system-critical workload
      priorityClassName: system-node-critical
      containers:
      - name: nvidia-device-plugin-ctr
        image: nvcr.io/nvidia/k8s-device-plugin:v0.14.3
        env:
        - name: NVIDIA_MIG_MONITOR_DEVICES
          value: all
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop: ["ALL"]
        volumeMounts:
        - name: device-plugin
          mountPath: /var/lib/kubelet/device-plugins
        - name: config
          mountPath: /config
      volumes:
      - name: device-plugin
        hostPath:
          path: /var/lib/kubelet/device-plugins
      - name: config
        configMap:
          name: nvidia-device-plugin-config

---
# Paso 4: Deployment de modelo usando MIG instance específica
apiVersion: apps/v1
kind: Deployment
metadata:
  name: model-inference-lightweight
  namespace: mlops-prod
spec:
  replicas: 3
  selector:
    matchLabels:
      app: model-inference
  template:
    metadata:
      labels:
        app: model-inference
    spec:
      containers:
      - name: model-server
        image: myregistry/lightweight-model:v1.0.0
        resources:
          requests:
            # Solicita MIG instance 1g.5gb (5GB GPU memory, 1/7 compute)
            nvidia.com/mig-1g.5gb: 1
            cpu: "2000m"
            memory: "4Gi"
          limits:
            nvidia.com/mig-1g.5gb: 1
            cpu: "4000m"
            memory: "8Gi"
        ports:
        - containerPort: 8080
        env:
        - name: CUDA_VISIBLE_DEVICES
          value: "0" # Kubernetes asigna automáticamente la MIG instance correcta

# RESULTADO: 3 replicas del modelo corriendo cada una en su MIG instance aislada
# Antes: 3 GPUs completas necesarias = $90k/año
# Después: 3 MIG instances en 1 GPU A100 = $30k/año
# Ahorro: 67% ($60k/año)

# Configuración GPU time-slicing para Kubernetes
# Permite múltiples pods compartir una GPU física mediante scheduling

apiVersion: v1
kind: ConfigMap
metadata:
  name: nvidia-device-plugin-config
  namespace: kube-system
data:
  # Configuración time-slicing: cada GPU física se expone como 4 "replicas" lógicas
  # Kubernetes puede asignar hasta 4 pods a la misma GPU física
  config.yaml: |
    version: v1
    sharing:
      timeSlicing:
        renameByDefault: false
        failRequestsGreaterThanOne: false
        resources:
        - name: nvidia.com/gpu
          replicas: 4 # Cada GPU física = 4 GPUs lógicas para scheduler

---
# Ejemplo: 3 modelos de inferencia lightweight comparten 1 GPU V100

apiVersion: apps/v1
kind: Deployment
metadata:
  name: model-a-inference
spec:
  replicas: 2
  template:
    spec:
      containers:
      - name: model-server
        image: myregistry/model-a:latest
        resources:
          limits:
            nvidia.com/gpu: 1 # Solicita 1 "logical" GPU (puede ser 1/4 de physical GPU)

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: model-b-inference
spec:
  replicas: 2
  template:
    spec:
      containers:
      - name: model-server
        image: myregistry/model-b:latest
        resources:
          limits:
            nvidia.com/gpu: 1

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: model-c-inference
spec:
  replicas: 2
  template:
    spec:
      containers:
      - name: model-server
        image: myregistry/model-c:latest
        resources:
          limits:
            nvidia.com/gpu: 1

# RESULTADO: 6 pods (2+2+2) corriendo en 2 GPUs V100 físicas mediante time-slicing
# Antes: 6 GPUs necesarias = $180k/año
# Después: 2 GPUs con time-slicing = $60k/año
# Ahorro: 67% ($120k/año)
# ⚠️ TRADE-OFF: Performance puede degradar si todos los pods usan GPU simultáneamente
# Monitoring crítico: track GPU utilization y latency P95 por pod

# Instalar Kubecost para GPU cost tracking

# Add Helm repo
helm repo add kubecost https://kubecost.github.io/cost-analyzer/
helm repo update

# Install Kubecost con Prometheus integration
helm install kubecost kubecost/cost-analyzer \
  --namespace kubecost \
  --create-namespace \
  --set prometheus.server.global.external_labels.cluster_id=mlops-prod-eks \
  --set kubecostToken="aGVsbG9Aa3ViZWNvc3QuY29t" \
  --set ingress.enabled=true \
  --set ingress.hosts[0]=kubecost.company.com

# Verificar instalación
kubectl get pods -n kubecost

# Acceder UI: http://kubecost.company.com (o port-forward)
kubectl port-forward -n kubecost svc/kubecost-cost-analyzer 9090:9090

# Métricas GPU disponibles en Kubecost:
# - GPU cost por pod (calculado desde instance type pricing)
# - GPU hours por namespace/label/team
# - GPU utilization % por pod
# - Idle GPU cost (allocated pero no usado)
# - Recommendations: "Rightsizing" (reduce GPU type si utilization 

# Error típico: Service no puede alcanzar pods del modelo de inferencia
# Problema: Selector labels no coinciden

apiVersion: v1
kind: Service
metadata:
  name: ml-model-inference
  namespace: mlops-prod
spec:
  selector:
    app: ml-model # ⚠️ Typo o label incorrecto
  ports:
  - protocol: TCP
    port: 80
    targetPort: 8000

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ml-model-inference
  namespace: mlops-prod
spec:
  replicas: 3
  selector:
    matchLabels:
      app: mlmodel-inference # ❌ No coincide con el Service selector
  template:
    metadata:
      labels:
        app: mlmodel-inference
    spec:
      containers:
      - name: model-server
        image: myregistry/ml-model:v1.2.3
        ports:
        - containerPort: 8000

# Este tipo de error no se detecta hasta deploy staging.
# Debugging toma 30-60 minutos porque "kubectl get svc" y "kubectl get pods"
# parecen OK pero el tráfico no llega.
# Solución: kubectl describe svc ml-model-inference (0 endpoints) 

# Configuración típica PersistentVolumeClaim para datos de entrenamiento ML
# ⚠️ Nota: StorageClass debe soportar ReadWriteMany si múltiples pods acceden

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: ml-training-data
  namespace: mlops-prod
spec:
  accessModes:
  - ReadWriteMany # Necesario para compartir entre preprocessing y training pods
  storageClassName: efs-sc # AWS EFS en este ejemplo (latencia mayor que EBS)
  resources:
    requests:
      storage: 500Gi # 500GB training dataset

---
# StorageClass para AWS EFS (ejemplo)

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: efs-sc
provisioner: efs.csi.aws.com
parameters:
  provisioningMode: efs-ap
  fileSystemId: fs-0123456789abcdef # ID del filesystem EFS pre-creado
  directoryPerms: "700"

# TRADE-OFFS:
# ✅ PRO: Múltiples pods pueden leer/escribir simultáneamente
# ❌ CON: Latencia 3-5x mayor que EBS gp3 (impacta training time 20-30%)
# ❌ CON: Coste 3x mayor que EBS por GB/mes
# ❌ CON: Single point of failure (si EFS falla, todo el pipeline para)

# ALTERNATIVA: S3 + Fuse mounts (s3fs) para read-only access desde training pods
# Mejor performance para datasets grandes pero requiere setup adicional

 

# Criteria para seleccionar workloads pilot migration

GOOD PILOT CANDIDATES (migra primero):
✓ Batch training jobs no customer-facing
✓ Experimentation workloads (Jupyter notebooks)
✓ Internal tools/dashboards (stakeholders tolerantes a downtime)
✓ Workloads con pocas dependencies externas
✓ Modelos ya containerizados (Docker images existen)
✓ Low traffic inference endpoints (

# Custom Prometheus exporter para métricas ML-specific
# Expone /metrics endpoint con latency, throughput, prediction confidence

from prometheus_client import Counter, Histogram, Gauge, start_http_server
import time
import numpy as np

# Definir métricas custom
prediction_latency = Histogram(
    'model_prediction_latency_seconds',
    'Latency de predicción del modelo (segundos)',
    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0]
)

prediction_counter = Counter(
    'model_predictions_total',
    'Total de predicciones realizadas',
    ['model_name', 'model_version', 'status']  # Labels para filtrar
)

prediction_confidence = Histogram(
    'model_prediction_confidence',
    'Distribución de confidence scores',
    buckets=[0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1.0]
)

active_requests = Gauge(
    'model_active_requests',
    'Número de requests siendo procesados actualmente'
)

model_load_time = Gauge(
    'model_load_time_seconds',
    'Tiempo para cargar modelo desde storage',
    ['model_name', 'model_version']
)

# Tu código de model serving con metrics instrumentadas
class ModelServer:
    def __init__(self, model_name: str, model_version: str):
        self.model_name = model_name
        self.model_version = model_version
        self.model = self.load_model()

    def load_model(self):
        """Carga modelo desde S3 y registra tiempo"""
        start = time.time()
        # ... código load model desde S3 ...
        model = load_model_from_s3(self.model_name, self.model_version)
        load_duration = time.time() - start

        model_load_time.labels(
            model_name=self.model_name,
            model_version=self.model_version
        ).set(load_duration)

        return model

    @prediction_latency.time()  # Decorator mide latency automáticamente
    def predict(self, features: np.ndarray) -> dict:
        """Ejecuta predicción y registra métricas"""
        active_requests.inc()  # Incrementa gauge al iniciar request

        try:
            # Ejecutar predicción
            start = time.time()
            predictions = self.model.predict(features)
            probabilities = self.model.predict_proba(features)

            # Extraer confidence (max probability por prediction)
            confidences = np.max(probabilities, axis=1)

            # Registrar métricas
            for conf in confidences:
                prediction_confidence.observe(conf)

            prediction_counter.labels(
                model_name=self.model_name,
                model_version=self.model_version,
                status='success'
            ).inc(len(features))

            return {
                'predictions': predictions.tolist(),
                'confidences': confidences.tolist(),
                'latency_ms': (time.time() - start) * 1000
            }

        except Exception as e:
            prediction_counter.labels(
                model_name=self.model_name,
                model_version=self.model_version,
                status='error'
            ).inc()
            raise e

        finally:
            active_requests.dec()  # Decrementa gauge al terminar

# Iniciar servidor HTTP para /metrics endpoint (puerto 8000)
if __name__ == '__main__':
    start_http_server(8000)  # Prometheus scraperá http://pod-ip:8000/metrics
    # Tu server FastAPI/Flask corre en puerto diferente (ej: 8080)
    model_server = ModelServer('fraud-detection', 'v2.3.1')
    # ... start FastAPI server on port 8080 ...

# ServiceMonitor para que Prometheus scrape métricas ML custom
# Usa kube-prometheus-stack (Prometheus Operator)

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: model-serving-metrics
  namespace: mlops-prod
  labels:
    release: prometheus # Label para que Prometheus Operator lo detecte
spec:
  selector:
    matchLabels:
      app: model-serving # Selecciona Services con este label
  endpoints:
  - port: metrics # Nombre del port en el Service
    path: /metrics
    interval: 15s # Scrape cada 15 segundos
    scrapeTimeout: 10s

---
# Service que expone el puerto de métricas

apiVersion: v1
kind: Service
metadata:
  name: model-serving
  namespace: mlops-prod
  labels:
    app: model-serving
spec:
  selector:
    app: model-serving
  ports:
  - name: http
    port: 8080 # Puerto FastAPI para predictions
    targetPort: 8080
  - name: metrics # Puerto para Prometheus scraping
    port: 8000
    targetPort: 8000

---
# PrometheusRule para alertas ML-specific

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: model-serving-alerts
  namespace: mlops-prod
  labels:
    release: prometheus
spec:
  groups:
  - name: ml_model_alerts
    interval: 30s
    rules:
    # Alerta: Latency P95 >500ms durante 10 minutos
    - alert: HighModelLatency
      expr: |
        histogram_quantile(0.95,
          rate(model_prediction_latency_seconds_bucket[5m])
        ) > 0.5
      for: 10m
      labels:
        severity: warning
        team: ml-platform
      annotations:
        summary: "Model {{ $labels.model_name }} high P95 latency"
        description: "P95 latency {{ $value }}s (threshold 0.5s)"

    # Alerta: Error rate >5% durante 5 minutos
    - alert: HighModelErrorRate
      expr: |
        sum(rate(model_predictions_total{status="error"}[5m]))
        /
        sum(rate(model_predictions_total[5m]))
        > 0.05
      for: 5m
      labels:
        severity: critical
        team: ml-platform
      annotations:
        summary: "Model {{ $labels.model_name }} high error rate"
        description: "Error rate {{ $value | humanizePercentage }}"

    # Alerta: Low confidence predictions >20% del total
    - alert: LowConfidencePredictions
      expr: |
        sum(rate(model_prediction_confidence_bucket{le="0.7"}[10m]))
        /
        sum(rate(model_prediction_confidence_count[10m]))
        > 0.2
      for: 15m
      labels:
        severity: warning
        team: ml-ops
      annotations:
        summary: "Model {{ $labels.model_name }} many low confidence predictions"
        description: "{{ $value | humanizePercentage }} predictions with confidence 

# Templates de deployment MLOps disponibles en self-service portal
# Data scientists seleccionan template, portal auto-genera Helm values + ejecuta deploy

templates:

# Template 1: Batch Training Job (Kubernetes Job)
- id: batch-training
  name: "Batch Model Training Job"
  description: "Entrena modelo ML en batch usando GPU, guarda artifacts en S3"
  icon: "🎓"
  parameters:
  - name: model_name
    type: string
    required: true
    description: "Nombre único del modelo (usado para tracking MLflow)"
  - name: dataset_s3_path
    type: string
    required: true
    description: "S3 path al dataset de entrenamiento (s3://bucket/path)"
  - name: gpu_type
    type: select
    options: ["g5.xlarge", "g5.2xlarge", "p4d.24xlarge"]
    default: "g5.xlarge"
    description: "Tipo de instancia GPU para training"
  - name: training_image
    type: string
    required: true
    description: "Docker image con código training (ECR URI)"
  - name: max_runtime_hours
    type: integer
    default: 24
    description: "Timeout máximo training job (horas)"

  # Backend auto-genera este Kubernetes Job manifest
  helm_chart: "mlops-platform/training-job"
  helm_values_template: |
    jobName: "{{ model_name }}-training-{{ timestamp }}"
    image: "{{ training_image }}"
    gpuType: "{{ gpu_type }}"
    datasetPath: "{{ dataset_s3_path }}"
    maxRuntimeSeconds: {{ max_runtime_hours * 3600 }}

# Template 2: Real-Time Model Serving (Kubernetes Deployment)
- id: model-serving
  name: "Real-Time Model Serving"
  description: "Despliega modelo para inferencia real-time con autoscaling"
  icon: "🚀"
  parameters:
  - name: model_name
    type: string
    required: true
  - name: model_s3_path
    type: string
    required: true
    description: "S3 path al model artifact (s3://bucket/model.pkl)"
  - name: min_replicas
    type: integer
    default: 2
    min: 1
    max: 10
    description: "Número mínimo de pods (HA)"
  - name: max_replicas
    type: integer
    default: 10
    min: 1
    max: 50
    description: "Número máximo de pods (autoscaling)"
  - name: target_latency_ms
    type: integer
    default: 100
    description: "Target P95 latency (ms) para autoscaling"
  - name: serving_framework
    type: select
    options: ["mlflow", "seldon", "kserve", "custom"]
    default: "mlflow"
    description: "Framework para model serving"

  helm_chart: "mlops-platform/model-serving"
  helm_values_template: |
    modelName: "{{ model_name }}"
    modelPath: "{{ model_s3_path }}"
    replicaCount: {{ min_replicas }}
    autoscaling:
      enabled: true
      minReplicas: {{ min_replicas }}
      maxReplicas: {{ max_replicas }}
      targetLatencyP95: {{ target_latency_ms }}
    framework: "{{ serving_framework }}"

# Template 3: Experiment Jupyter Notebook (Kubernetes Pod)
- id: jupyter-notebook
  name: "Jupyter Notebook Experiment"
  description: "Spin up Jupyter notebook con GPU para experimentación"
  icon: "📓"
  parameters:
  - name: notebook_name
    type: string
    required: true
  - name: gpu_enabled
    type: boolean
    default: true
    description: "Asignar GPU al notebook"
  - name: persistent_storage_gb
    type: integer
    default: 50
    min: 10
    max: 500
    description: "Tamaño PersistentVolume para guardar notebooks/data"
  - name: python_packages
    type: text
    default: "pandas\nscikit-learn\ntensorflow==2.14"
    description: "Requirements.txt packages (uno por línea)"

  helm_chart: "mlops-platform/jupyter-notebook"
  # Portal crea PVC + Deployment + Service automáticamente

# Workflow cuando data scientist usa template:
# 1. Selecciona template en UI
# 2. Completa form con parameters
# 3. Click "Deploy"
# 4. Portal valida inputs
# 5. Portal genera helm values desde template
# 6. Portal ejecuta: helm install  -f generated-values.yaml
# 7. Portal muestra logs deployment en tiempo real
# 8. Portal notifica cuando deployment ready (Slack/email)
# 9. Portal muestra URLs acceso (Jupyter, model endpoint, etc)

# Tiempo total: 4-6 minutos desde click "Deploy" hasta servicio running
# Antes (manual): 2-3 semanas incluyendo DevOps tickets, reviews, debugging

 

terraform/
├── environments/
│   ├── dev/
│   │   ├── main.tf           # Importa módulos con configuración dev
│   │   ├── variables.tf      # Variables específicas dev
│   │   ├── terraform.tfvars  # Valores dev (instance types pequeños, 1 AZ)
│   │   └── backend.tf        # S3 backend para state dev
│   ├── staging/
│   │   └── [mismo patrón]
│   └── prod/
│       └── [mismo patrón, multi-AZ, instance types grandes]
│
├── modules/
│   ├── mlops-cluster/        # EKS cluster con GPU support
│   │   ├── main.tf           # Define EKS cluster, node groups
│   │   ├── variables.tf      # Input variables (cluster_name, region, k8s_version)
│   │   ├── outputs.tf        # Output cluster endpoint, OIDC provider
│   │   └── README.md
│   │
│   ├── mlops-networking/     # VPC, subnets, security groups
│   │   ├── main.tf           # VPC con private/public subnets
│   │   ├── variables.tf
│   │   └── outputs.tf
│   │
│   ├── mlops-storage/        # S3 buckets para artifacts, EFS para shared data
│   │   ├── main.tf           # S3 buckets con encryption, lifecycle policies
│   │   ├── variables.tf
│   │   └── outputs.tf
│   │
│   ├── mlops-iam/            # IAM roles para service accounts (IRSA)
│   │   ├── main.tf           # Roles para MLflow, model serving pods
│   │   ├── variables.tf
│   │   └── outputs.tf
│   │
│   └── mlops-monitoring/     # Prometheus, Grafana, Loki via Helm
│       ├── main.tf           # Helm releases usando Terraform Helm provider
│       ├── variables.tf
│       └── outputs.tf
│
├── global/
│   ├── backend.tf            # Configuración S3 backend + DynamoDB locking
│   └── providers.tf          # Proveedores AWS, Kubernetes, Helm
│
└── README.md                 # Documentación arquitectura + runbook deployment 

# Módulo Terraform para EKS Cluster MLOps con GPU support
# Características: Managed node groups, autoscaling, IRSA, logging

terraform {
required_version = ">= 1.6"
required_providers {
    aws = {
    source  = "hashicorp/aws"
    version = "~> 5.0"
    }
    kubernetes = {
    source  = "hashicorp/kubernetes"
    version = "~> 2.20"
    }
}
}

# EKS Cluster principal
resource "aws_eks_cluster" "mlops" {
name     = var.cluster_name
role_arn = aws_iam_role.cluster.arn
version  = var.kubernetes_version

vpc_config {
    subnet_ids              = var.subnet_ids
    endpoint_private_access = true
    endpoint_public_access  = var.enable_public_access # False para prod
    security_group_ids      = [aws_security_group.cluster.id]
}

enabled_cluster_log_types = [
    "api",
    "audit",
    "authenticator",
    "controllerManager",
    "scheduler"
]

# Encryption para secrets en etcd
encryption_config {
    provider {
    key_arn = var.kms_key_arn
    }
    resources = ["secrets"]
}

depends_on = [
    aws_iam_role_policy_attachment.cluster_AmazonEKSClusterPolicy,
    aws_cloudwatch_log_group.cluster
]

tags = merge(
    var.common_tags,
    {
    Name        = var.cluster_name
    Environment = var.environment
    Purpose     = "MLOps"
    }
)
}

# Node Group para cargas de trabajo CPU (system pods, MLflow, monitoring)
resource "aws_eks_node_group" "cpu" {
cluster_name    = aws_eks_cluster.mlops.name
node_group_name = "${var.cluster_name}-cpu"
node_role_arn   = aws_iam_role.node.arn
subnet_ids      = var.private_subnet_ids

scaling_config {
    desired_size = var.cpu_node_desired_size
    max_size     = var.cpu_node_max_size
    min_size     = var.cpu_node_min_size
}

instance_types = var.cpu_instance_types # Ejemplo: ["m5.2xlarge", "m5a.2xlarge"]
capacity_type  = "ON_DEMAND"            # SPOT para ambientes no-prod

update_config {
    max_unavailable = 1 # Rolling updates sin downtime
}

labels = {
    workload = "cpu"
    role     = "mlops-cpu"
}

tags = merge(
    var.common_tags,
    {
    Name = "${var.cluster_name}-cpu-nodes"
    }
)

depends_on = [
    aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy,
    aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy,
    aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly,
]
}

# Node Group para cargas de trabajo GPU (training, inference)
resource "aws_eks_node_group" "gpu" {
cluster_name    = aws_eks_cluster.mlops.name
node_group_name = "${var.cluster_name}-gpu"
node_role_arn   = aws_iam_role.node.arn
subnet_ids      = var.private_subnet_ids

scaling_config {
    desired_size = var.gpu_node_desired_size
    max_size     = var.gpu_node_max_size
    min_size     = var.gpu_node_min_size
}

# g5.xlarge: 1x NVIDIA A10G GPU, 4 vCPU, 16GB RAM - $1.006/hr spot
# g5.2xlarge: 1x A10G, 8 vCPU, 32GB RAM - $1.212/hr spot
# p4d.24xlarge: 8x A100 GPUs para training masivo - $32.77/hr on-demand
instance_types = var.gpu_instance_types # Ejemplo: ["g5.xlarge", "g5.2xlarge"]
capacity_type  = var.gpu_capacity_type  # "SPOT" ahorra 70% vs on-demand

update_config {
    max_unavailable = 1
}

# AMI optimizada para GPU con NVIDIA drivers pre-instalados
ami_type = "AL2_x86_64_GPU"

labels = {
    workload        = "gpu"
    role            = "mlops-gpu"
    "nvidia.com/gpu" = "true" # Usado por GPU scheduler
}

# Taint para asegurar solo pods que requieren GPU se schedulean aquí
taint {
    key    = "nvidia.com/gpu"
    value  = "true"
    effect = "NO_SCHEDULE"
}

tags = merge(
    var.common_tags,
    {
    Name = "${var.cluster_name}-gpu-nodes"
    }
)

depends_on = [
    aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy,
    aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy,
    aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly,
]
}

# OIDC Provider para IRSA (IAM Roles for Service Accounts)
# Permite pods Kubernetes asumir IAM roles sin access keys hardcoded
resource "aws_iam_openid_connect_provider" "cluster" {
client_id_list  = ["sts.amazonaws.com"]
thumbprint_list = [data.tls_certificate.cluster.certificates[0].sha1_fingerprint]
url             = aws_eks_cluster.mlops.identity[0].oidc[0].issuer

tags = var.common_tags
}

data "tls_certificate" "cluster" {
url = aws_eks_cluster.mlops.identity[0].oidc[0].issuer
}

# CloudWatch Log Group para cluster logs
resource "aws_cloudwatch_log_group" "cluster" {
name              = "/aws/eks/${var.cluster_name}/cluster"
retention_in_days = var.log_retention_days # 7 días dev, 30 días prod
kms_key_id        = var.kms_key_arn

tags = var.common_tags
}

# Outputs útiles para otros módulos
output "cluster_id" {
description = "EKS cluster ID"
value       = aws_eks_cluster.mlops.id
}

output "cluster_endpoint" {
description = "Endpoint para kubectl"
value       = aws_eks_cluster.mlops.endpoint
}

output "cluster_certificate_authority_data" {
description = "CA cert para autenticar con cluster"
value       = aws_eks_cluster.mlops.certificate_authority[0].data
sensitive   = true
}

output "oidc_provider_arn" {
description = "ARN del OIDC provider para IRSA"
value       = aws_iam_openid_connect_provider.cluster.arn
}

# Umbrella Chart para MLOps Platform completa
# Un solo `helm install` despliega: MLflow, Kubeflow Pipelines, Prometheus, Grafana

apiVersion: v2
name: mlops-platform
description: Production-ready MLOps platform for Kubernetes
type: application
version: 1.0.0        # Chart version (semantic versioning)
appVersion: "2024.1"  # Version de la plataforma MLOps

# Dependencies = subcharts que Helm instalará automáticamente
dependencies:
  # MLflow Tracking Server para experiment tracking
  - name: mlflow
    version: "0.7.19"
    repository: "https://community-charts.github.io/helm-charts"
    condition: mlflow.enabled  # Se instala solo si mlflow.enabled: true en values.yaml

  # PostgreSQL para MLflow backend store
  - name: postgresql
    version: "12.x.x"
    repository: "https://charts.bitnami.com/bitnami"
    condition: postgresql.enabled

  # MinIO como artifact store (alternativa a S3 para on-prem)
  - name: minio
    version: "5.0.x"
    repository: "https://charts.min.io/"
    condition: minio.enabled

  # Kubeflow Pipelines para ML workflows orchestration
  - name: kubeflow-pipelines
    version: "2.0.3"
    repository: "https://charts.kubeflow.org"
    condition: kubeflow.enabled

  # Prometheus + Grafana para monitoring
  - name: kube-prometheus-stack
    version: "51.x.x"
    repository: "https://prometheus-community.github.io/helm-charts"
    condition: monitoring.enabled

  # Seldon Core para model serving avanzado
  - name: seldon-core-operator
    version: "1.17.x"
    repository: "https://storage.googleapis.com/seldon-charts"
    condition: seldon.enabled

# Maintainers info
maintainers:
  - name: Abdessamad Ammi
    email: sam@bcloud.consulting

# Configuración production para MLOps Platform
# Usa: helm install mlops-platform . -f values-prod.yaml

# MLflow Tracking Server
mlflow:
  enabled: true
  replicaCount: 3  # Alta disponibilidad

  image:
    repository: ghcr.io/mlflow/mlflow
    tag: "2.9.2"

  resources:
    requests:
      cpu: "500m"
      memory: "1Gi"
    limits:
      cpu: "2000m"
      memory: "4Gi"

  # Backend store = PostgreSQL para metadata
  backendStore:
    postgres:
      enabled: true
      host: "postgresql.mlops-platform.svc.cluster.local"
      port: 5432
      database: "mlflow"
      username: "mlflow"
      # Password desde Kubernetes Secret (nunca hardcodear)
      passwordSecret: "mlflow-postgres-secret"

  # Artifact store = S3 para model artifacts
  artifactStore:
    type: "s3"
    s3:
      bucket: "mlops-prod-artifacts"
      region: "us-east-1"
      # IRSA: Pod asume IAM role, no access keys hardcoded
      serviceAccount:
        create: true
        annotations:
          eks.amazonaws.com/role-arn: "arn:aws:iam::123456789012:role/mlflow-s3-access"

  ingress:
    enabled: true
    className: "nginx"
    hosts:
      - host: mlflow.company.com
        paths:
          - path: /
            pathType: Prefix
    tls:
      - secretName: mlflow-tls
        hosts:
          - mlflow.company.com

# PostgreSQL database
postgresql:
  enabled: true
  auth:
    username: mlflow
    database: mlflow
    # Password auto-generado por Helm, stored in Secret
    existingSecret: "mlflow-postgres-secret"

  primary:
    persistence:
      enabled: true
      storageClass: "gp3"  # AWS EBS gp3 para mejor IOPS
      size: 100Gi

    resources:
      requests:
        cpu: "1000m"
        memory: "2Gi"
      limits:
        cpu: "4000m"
        memory: "8Gi"

# MinIO (S3-compatible object storage para on-prem)
minio:
  enabled: false  # Usamos S3 real en prod, MinIO para dev/staging

# Kubeflow Pipelines
kubeflow:
  enabled: true
  # Configuración específica Kubeflow (simplificado)

# Prometheus + Grafana Monitoring Stack
monitoring:
  enabled: true

  prometheus:
    prometheusSpec:
      retention: 30d  # Retención métricas 30 días
      storageSpec:
        volumeClaimTemplate:
          spec:
            storageClassName: gp3
            resources:
              requests:
                storage: 200Gi

    # Service Monitors para scrape ML-specific metrics
    additionalServiceMonitors:
      - name: mlflow-metrics
        selector:
          matchLabels:
            app: mlflow
        endpoints:
          - port: http
            path: /metrics

  grafana:
    enabled: true
    adminPassword: "changeme"  # Cambiar por Secret en producción

    persistence:
      enabled: true
      storageClassName: gp3
      size: 10Gi

    # Pre-load dashboards MLOps
    dashboardProviders:
      dashboardproviders.yaml:
        apiVersion: 1
        providers:
          - name: 'mlops'
            folder: 'MLOps'
            type: file
            options:
              path: /var/lib/grafana/dashboards/mlops

# Seldon Core para model serving
seldon:
  enabled: true
  # Configuración específica Seldon 

# Terraform gestiona Helm releases dentro del cluster EKS
# Ventaja: Un solo `terraform apply` provisiona cluster Y despliega aplicaciones

terraform {
  required_providers {
    helm = {
      source  = "hashicorp/helm"
      version = "~> 2.11"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.20"
    }
  }
}

# Provider Helm apunta al cluster EKS creado previamente
provider "helm" {
  kubernetes {
    host                   = module.mlops_cluster.cluster_endpoint
    cluster_ca_certificate = base64decode(module.mlops_cluster.cluster_certificate_authority_data)
    token                  = data.aws_eks_cluster_auth.cluster.token
  }
}

# Namespace para MLOps platform
resource "kubernetes_namespace" "mlops_platform" {
  metadata {
    name = "mlops-platform"
    labels = {
      name        = "mlops-platform"
      environment = var.environment
    }
  }
}

# Helm release: MLflow
resource "helm_release" "mlflow" {
  name       = "mlflow"
  repository = "https://community-charts.github.io/helm-charts"
  chart      = "mlflow"
  version    = "0.7.19"
  namespace  = kubernetes_namespace.mlops_platform.metadata[0].name

  # Values desde archivo YAML
  values = [
    file("${path.module}/helm-values/mlflow-${var.environment}.yaml")
  ]

  # Wait for all resources to be ready
  wait    = true
  timeout = 600 # 10 minutos timeout

  # Dependencias: PostgreSQL debe estar ready antes
  depends_on = [helm_release.postgresql]
}

# Helm release: PostgreSQL
resource "helm_release" "postgresql" {
  name       = "postgresql"
  repository = "https://charts.bitnami.com/bitnami"
  chart      = "postgresql"
  version    = "12.12.10"
  namespace  = kubernetes_namespace.mlops_platform.metadata[0].name

  set {
    name  = "auth.username"
    value = "mlflow"
  }

  set {
    name  = "auth.database"
    value = "mlflow"
  }

  # Password desde Terraform sensitive variable (mejor: AWS Secrets Manager)
  set_sensitive {
    name  = "auth.password"
    value = var.mlflow_db_password
  }

  set {
    name  = "primary.persistence.size"
    value = var.environment == "prod" ? "100Gi" : "20Gi"
  }

  wait    = true
  timeout = 600
}

# Helm release: Prometheus + Grafana monitoring stack
resource "helm_release" "prometheus_stack" {
  name       = "prometheus"
  repository = "https://prometheus-community.github.io/helm-charts"
  chart      = "kube-prometheus-stack"
  version    = "51.9.4"
  namespace  = kubernetes_namespace.mlops_platform.metadata[0].name

  values = [
    file("${path.module}/helm-values/prometheus-${var.environment}.yaml")
  ]

  # Configuraciones específicas via set
  set {
    name  = "prometheus.prometheusSpec.retention"
    value = var.environment == "prod" ? "30d" : "7d"
  }

  set {
    name  = "prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage"
    value = var.environment == "prod" ? "200Gi" : "50Gi"
  }

  wait    = true
  timeout = 900 # Stack complejo, 15 min timeout
}

# Output URLs útiles
output "mlflow_url" {
  description = "MLflow Tracking UI URL"
  value       = "https://mlflow.${var.domain_name}"
}

output "grafana_url" {
  description = "Grafana dashboards URL"
  value       = "https://grafana.${var.domain_name}"
}