GitOps: Infrastructure and Application Delivery via Git

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

# End-to-End GitOps Workflow
 
1. DEVELOP
   - Developer creates feature branch in app repository
   - Writes code, tests locally
   - Opens pull request
 
2. REVIEW
   - Peers review code changes
   - Automated checks run (linting, unit tests)
   - Security scanning on dependencies
   - Approval required before merge
 
3. BUILD (CI Pipeline)
   - Merge triggers CI pipeline
   - Build application
   - Run full test suite
   - Scan for vulnerabilities
   - Build container image
   - Push to registry with commit SHA tag
     Example: registry.example.com/my-app:a1b2c3d
 
4. UPDATE CONFIG (Automated or Manual)
   Option A - Automated (recommended for dev/staging):
   - CI updates image tag in config repo
   - Commits directly to development branch
   - GitOps operator detects and deploys
   
   Option B - Pull Request (recommended for production):
   - CI opens PR to update production image tag
   - Requires human approval
   - Merge triggers deployment
 
5. RECONCILE
   - GitOps operator detects config change
   - Renders manifests (Kustomize/Helm)
   - Computes diff against live state
   - Applies changes to cluster
   - Waits for health check
 
6. VERIFY
   - Deployment rolls out new pods
   - Health checks pass
   - Metrics confirm normal operation
   - Alerts if anomalies detected
 
7. ROLLBACK (if needed)
   - Revert commit in config repo
   - OR: Update image tag to previous version
   - GitOps operator applies reverted state
   - Deployment rolls back

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

# .github/workflows/promote-to-staging.yml
name: Promote to Staging
 
on:
  # Trigger when dev deployment succeeds
  workflow_run:
    workflows: ["Deploy to Development"]
    types: [completed]
    branches: [main]
 
jobs:
  promote:
    runs-on: ubuntu-latest
    if: ${{ github.event.workflow_run.conclusion == 'success' }}
    
    steps:
      - uses: actions/checkout@v4
        with:
          repository: company/infrastructure
          token: ${{ secrets.GH_PAT }}
      
      - name: Get current dev image
        id: dev-image
        run: |
          IMAGE=$(yq '.images[0].newTag' \
            apps/my-app/overlays/development/kustomization.yaml)
          echo "tag=$IMAGE" >> $GITHUB_OUTPUT
      
      - name: Update staging image
        run: |
          cd apps/my-app/overlays/staging
          kustomize edit set image my-app=registry.example.com/my-app:${{ steps.dev-image.outputs.tag }}
      
      - name: Create promotion PR
        uses: peter-evans/create-pull-request@v5
        with:
          token: ${{ secrets.GH_PAT }}
          commit-message: "promote: my-app ${{ steps.dev-image.outputs.tag }} to staging"
          title: "Promote my-app to staging"
          body: |
            ## Promotion Request
            
            Promoting my-app to staging environment.
            
            **Image Tag**: ${{ steps.dev-image.outputs.tag }}
            **Source**: Development (verified healthy)
            
            ### Checklist
            - [ ] Development deployment verified
            - [ ] No breaking changes in release notes
            - [ ] Ready for staging validation
          branch: promote/my-app-staging
          labels: promotion,staging

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

# ApplicationSet for preview environments
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: preview-environments
  namespace: argocd
spec:
  generators:
    # Generate from pull requests
    - pullRequest:
        github:
          owner: company
          repo: my-app
          tokenRef:
            secretName: github-token
            key: token
        requeueAfterSeconds: 60
  
  template:
    metadata:
      name: 'preview-{{branch}}-{{number}}'
      labels:
        app.kubernetes.io/instance: preview
        preview-pr: '{{number}}'
    spec:
      project: previews
      
      source:
        repoURL: https://github.com/company/infrastructure.git
        targetRevision: HEAD
        path: apps/my-app/overlays/preview
        kustomize:
          images:
            - my-app=registry.example.com/my-app:pr-{{head_sha}}
          nameSuffix: '-{{number}}'
      
      destination:
        server: https://kubernetes.default.svc
        namespace: 'preview-{{number}}'
      
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true
      
      # Auto-delete after PR closure
      syncPolicy:
        automated:
          prune: true

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

# Argo Rollout with Canary Strategy
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: my-app
spec:
  replicas: 10
  
  selector:
    matchLabels:
      app: my-app
  
  template:
    metadata:
      labels:
        app: my-app
    spec:
      containers:
        - name: my-app
          image: my-app:v1.5.3
          ports:
            - containerPort: 8080
  
  strategy:
    canary:
      # Traffic routing via service mesh or ingress
      trafficRouting:
        nginx:
          stableIngress: my-app-stable
          
      # Canary analysis with Prometheus metrics
      analysis:
        templates:
          - templateName: success-rate
        startingStep: 2
        args:
          - name: service-name
            value: my-app
      
      # Progressive rollout steps
      steps:
        # Step 1: 5% traffic to canary
        - setWeight: 5
        - pause: {duration: 5m}
        
        # Step 2: 10% traffic, start analysis
        - setWeight: 10
        - pause: {duration: 10m}
        
        # Step 3: 25% traffic
        - setWeight: 25
        - pause: {duration: 10m}
        
        # Step 4: 50% traffic
        - setWeight: 50
        - pause: {duration: 15m}
        
        # Step 5: 100% - full promotion
        - setWeight: 100
 
---
# Analysis Template using Prometheus
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: success-rate
spec:
  args:
    - name: service-name
  
  metrics:
    - name: success-rate
      interval: 1m
      # Must stay above 95% success rate
      successCondition: result[0] >= 0.95
      failureLimit: 3
      
      provider:
        prometheus:
          address: http://prometheus.monitoring:9090
          query: |
            sum(rate(
              http_requests_total{
                service="{{args.service-name}}",
                status=~"2.."
              }[5m]
            )) /
            sum(rate(
              http_requests_total{
                service="{{args.service-name}}"
              }[5m]
            ))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

# Require images from approved registries
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-approved-registry
spec:
  validationFailureAction: Enforce
  background: false
  rules:
    - name: check-registry
      match:
        any:
          - resources:
              kinds:
                - Pod
      validate:
        message: "Images must come from approved registries"
        pattern:
          spec:
            containers:
              - image: "registry.example.com/* | gcr.io/my-project/*"
 
---
# Require resource limits
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-resource-limits
spec:
  validationFailureAction: Enforce
  rules:
    - name: check-limits
      match:
        any:
          - resources:
              kinds:
                - Deployment
                - StatefulSet
      validate:
        message: "All containers must have resource limits"
        pattern:
          spec:
            template:
              spec:
                containers:
                  - resources:
                      limits:
                        memory: "?*"
                        cpu: "?*"
 
---
# Require labels for GitOps traceability
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-labels
spec:
  validationFailureAction: Enforce
  rules:
    - name: check-labels
      match:
        any:
          - resources:
              kinds:
                - Deployment
                - Service
      validate:
        message: "Resources must have required labels"
        pattern:
          metadata:
            labels:
              app.kubernetes.io/name: "?*"
              app.kubernetes.io/managed-by: "?*"