Terraform grafana_slo로 SLO-as-Code 구현하기: GitOps 파이프라인 단계별 구현

# grafana_queries 타입: 기존 대시보드 패널 쿼리를 SLI로 재활용
resource "grafana_slo" "panel_query_slo" {
  name        = "Checkout API Availability (Dashboard Query)"
  description = "기존 대시보드 패널 쿼리를 SLI로 재활용"
 
  query {
    type = "grafana_queries"
    grafana_queries {
      success_query {
        query_type = "instant"
        ref_id     = "A"
        expr       = "sum(http_requests_total{service=\"checkout\",status!~\"5..\"})"
      }
      total_query {
        query_type = "instant"
        ref_id     = "B"
        expr       = "sum(http_requests_total{service=\"checkout\"})"
      }
    }
  }
 
  objectives {
    value  = 0.999
    window = "30d"
  }
 
  destination_datasource {
    uid = var.datasource_uid
  }
}

terraform {
  required_providers {
    grafana = {
      source  = "grafana/grafana"
      version = ">= 3.5.0"
    }
  }
}
 
provider "grafana" {
  url  = var.grafana_url
  auth = var.grafana_service_account_token
}
 
# 데이터소스를 동적으로 참조 (하드코딩 대신 권장)
data "grafana_data_source" "prometheus" {
  name = "grafanacloud-prom"
}
 
# Ratio 방식: HTTP 가용성 SLO
resource "grafana_slo" "api_availability" {
  name        = "API Availability SLO"
  description = "HTTP 5xx 제외 성공 요청 비율"
 
  query {
    type = "ratio"
    ratio {
      success_metric  = "http_requests_total{status!~\"5..\"}"
      total_metric    = "http_requests_total"
      group_by_labels = ["service", "env"]
    }
  }
 
  objectives {
    value  = 0.999
    window = "30d"
  }
 
  destination_datasource {
    uid = data.grafana_data_source.prometheus.uid
  }
 
  label {
    key   = "team"
    value = "platform"
  }
 
  alerting {
    fastburn {
      annotation {
        key   = "summary"
        value = "에러 버짓 급속 소진 중"
      }
      label {
        key   = "severity"
        value = "critical"
      }
    }
    slowburn {
      annotation {
        key   = "summary"
        value = "에러 버짓 지속 소진 중"
      }
      label {
        key   = "severity"
        value = "warning"
      }
    }
  }
}

# Freeform 방식: P99 지연시간 SLO
resource "grafana_slo" "latency_slo" {
  name        = "API P99 지연시간 SLO"
  description = "P99 응답시간 200ms 이하 유지"
 
  query {
    type = "freeform"
    freeform {
      query = "sum(rate(http_request_duration_seconds_bucket{le=\"0.2\"}[$__rate_interval])) / sum(rate(http_request_duration_seconds_count[$__rate_interval]))"
    }
  }
 
  objectives {
    value  = 0.95
    window = "7d"
  }
 
  destination_datasource {
    uid = data.grafana_data_source.prometheus.uid
  }
}

slo-gitops/
├── modules/
│   └── grafana-slo/
│       ├── main.tf        # grafana_slo 리소스 정의
│       ├── variables.tf   # 입력 변수 선언
│       └── outputs.tf     # 출력값 선언
├── envs/
│   ├── staging/
│   │   ├── backend.tf     # S3 원격 상태 설정
│   │   ├── main.tf
│   │   └── terraform.tfvars
│   └── production/
│       ├── backend.tf
│       ├── main.tf
│       └── terraform.tfvars
└── .github/
    └── workflows/
        └── slo-deploy.yml

variable "service_name" {
  type        = string
  description = "SLO 이름에 사용할 서비스 식별자"
}
 
variable "team" {
  type        = string
  description = "담당 팀 이름 (레이블 및 설명에 사용)"
}
 
variable "slo_target" {
  type        = number
  default     = 0.999
  description = "SLO 목표값 (0.0 ~ 1.0)"
}
 
variable "success_metric" {
  type        = string
  description = "성공 요청을 나타내는 PromQL 메트릭"
}
 
variable "total_metric" {
  type        = string
  description = "전체 요청을 나타내는 PromQL 메트릭"
}
 
variable "datasource_uid" {
  type        = string
  description = "Prometheus 데이터소스 UID"
}

resource "grafana_slo" "this" {
  name        = "${var.service_name} Availability"
  description = "Managed by Terraform - team: ${var.team}"
 
  query {
    type = "ratio"
    ratio {
      success_metric = var.success_metric
      total_metric   = var.total_metric
    }
  }
 
  objectives {
    value  = var.slo_target
    window = "30d"
  }
 
  destination_datasource {
    uid = var.datasource_uid
  }
 
  label {
    key   = "team"
    value = var.team
  }
  label {
    key   = "managed"
    value = "terraform"
  }
 
  alerting {
    fastburn {
      label {
        key   = "severity"
        value = "critical"
      }
    }
    slowburn {
      label {
        key   = "severity"
        value = "warning"
      }
    }
  }
}

output "slo_id" {
  value       = grafana_slo.this.id
  description = "생성된 SLO의 UUID"
}
 
output "slo_name" {
  value       = grafana_slo.this.name
  description = "생성된 SLO의 이름"
}

terraform {
  backend "s3" {
    bucket  = "my-terraform-state"
    key     = "slo/production/terraform.tfstate"
    region  = "ap-northeast-2"
    encrypt = true
 
    # Terraform 1.9 이하: DynamoDB 잠금 사용
    dynamodb_table = "terraform-lock"
 
    # Terraform 1.10+: 네이티브 S3 잠금 (위 dynamodb_table 대신 사용 가능)
    # use_lockfile = true
  }
}

data "grafana_data_source" "prometheus" {
  name = "grafanacloud-prom"
}
 
module "checkout_slo" {
  source = "../../modules/grafana-slo"
 
  service_name   = "checkout-api"
  team           = "payments"
  slo_target     = 0.999
  success_metric = "http_requests_total{service=\"checkout\",status!~\"5..\"}"
  total_metric   = "http_requests_total{service=\"checkout\"}"
  datasource_uid = data.grafana_data_source.prometheus.uid
}
 
module "auth_slo" {
  source = "../../modules/grafana-slo"
 
  service_name   = "auth-service"
  team           = "identity"
  slo_target     = 0.9995
  success_metric = "http_requests_total{service=\"auth\",status!~\"5..\"}"
  total_metric   = "http_requests_total{service=\"auth\"}"
  datasource_uid = data.grafana_data_source.prometheus.uid
}

name: SLO Deploy
 
on:
  push:
    branches: [main]
    paths: ["envs/**", "modules/**"]
  pull_request:
    branches: [main]
    paths: ["envs/**", "modules/**"]
 
env:
  TF_VAR_grafana_url: ${{ secrets.GRAFANA_URL }}
  TF_VAR_grafana_token: ${{ secrets.GRAFANA_SERVICE_ACCOUNT_TOKEN }}
 
jobs:
  plan:
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    strategy:
      matrix:
        env: [staging, production]
    steps:
      - uses: actions/checkout@v4
 
      - uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: "1.9.x"
 
      - name: Configure AWS Credentials (S3 backend)
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: ap-northeast-2
 
      - name: Terraform Init
        run: terraform init
        working-directory: envs/${{ matrix.env }}
 
      - name: Terraform Plan
        id: plan
        run: |
          terraform plan -no-color -out=tfplan
          terraform show -no-color tfplan > plan_output.txt
        working-directory: envs/${{ matrix.env }}
 
      - name: Comment Plan on PR
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            let output = fs.readFileSync('envs/${{ matrix.env }}/plan_output.txt', 'utf8');
            // GitHub PR 코멘트는 65,536자 제한이 있으므로 초과 시 잘라냄
            if (output.length > 60000) {
              output = output.substring(0, 60000) + '\n...(출력이 길어 잘렸습니다. 전체 내용은 Actions 로그를 확인하세요)';
            }
            const body = `#### Terraform Plan — ${{ matrix.env }}\n\`\`\`\n${output}\n\`\`\``;
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: body
            });
 
  # Staging 먼저 적용
  apply-staging:
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
    environment: staging
    steps:
      - uses: actions/checkout@v4
      - uses: hashicorp/setup-terraform@v3
 
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: ap-northeast-2
 
      - run: terraform init
        working-directory: envs/staging
 
      - run: terraform apply -auto-approve
        working-directory: envs/staging
 
  # Staging 성공 후 Production 적용 (순차 배포)
  apply-production:
    runs-on: ubuntu-latest
    needs: apply-staging
    environment: production
    steps:
      - uses: actions/checkout@v4
      - uses: hashicorp/setup-terraform@v3
 
      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: ap-northeast-2
 
      - run: terraform init
        working-directory: envs/production
 
      - run: terraform apply -auto-approve
        working-directory: envs/production

# 1단계: 기존 SLO를 HCL 형식으로 내보내기
curl -H "Authorization: Bearer $GRAFANA_TOKEN" \
  "https://your-org.grafana.net/api/plugins/grafana-slo-app/resources/v1/slo?format=hcl" \
  -o existing_slos.tf
 
# 또는 Grafana UI에서: SLO 상세 페이지 → More > Export
 
# 2단계: Terraform state에 기존 SLO 등록 (재생성 없이 관리 시작)
terraform import grafana_slo.api_availability <SLO_UUID>
 
# 3단계: plan으로 코드와 실제 상태의 drift 확인
terraform plan

# grafana/grafana provider >= 3.7.0 필요, Asserts 기능 활성화 환경에서만 동작
resource "grafana_slo" "entity_slo" {
  name        = "Checkout API Availability (Knowledge Graph)"
  description = "Knowledge Graph 연동 SLO"
 
  query {
    type = "ratio"
    ratio {
      success_metric = "http_requests_total{service=\"checkout\",status!~\"5..\"}"
      total_metric   = "http_requests_total{service=\"checkout\"}"
    }
  }
 
  objectives {
    value  = 0.999
    window = "30d"
  }
 
  destination_datasource {
    uid = data.grafana_data_source.prometheus.uid
  }
 
  label {
    key   = "grafana_slo_provenance"
    value = "asserts"
  }
 
  # 연결할 서비스 엔티티 지정 (provider >= 3.7.0, Asserts 활성화 필요)
  search_expression = "service.name=\"checkout-api\" AND environment=\"production\""
}