Skip to content

Commit b74a992

Browse files
hamzabouissihamzabouissi
authored
major: migrating from pomerium to oauth2proxy (#690)
1 parent a995ba5 commit b74a992

13 files changed

+234
-120
lines changed

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,8 @@ This chart deploys the GlueOps Platform
8383
| dex.github.client_secret | string | `"placeholder_dex_github_client_secret"` | To create a clientSecret please reference: https://github.com/GlueOps/github-oauth-apps/tree/v0.0.1 |
8484
| dex.github.orgs | list | `["placeholder_admin_github_org_name","placeholder_tenant_github_org_name"]` | Specify the github orgs you want to allow access to. This is a list of strings. Note: users still need to be in the proper groups to have access. |
8585
| dex.grafana.client_secret | string | `"placeholder_dex_grafana_client_secret"` | Specify a unique password here. This will be used to connect grafana via OAuth to the Dex IDP. You can create one with in bash `openssl rand -base64 32` |
86-
| dex.pomerium.client_secret | string | `"placeholder_dex_pomerium_client_secret"` | Specify a unique password here. This will be used to connect argocd via OIDC to the Dex IDP. You can create one with in bash `openssl rand -base64 32` |
86+
| dex.oauth2.client_secret | string | `"placeholder_dex_oauth2_client_secret"` | |
87+
| dex.oauth2.cookie_secret | string | `"placeholder_dex_oauth2_cookie_secret"` | |
8788
| dex.vault.client_secret | string | `"placeholder_dex_vault_client_secret"` | |
8889
| externalDns.aws_accessKey | string | `"placeholder_externaldns_aws_access_key"` | Part of `externaldns_iam_credentials` output from terraform-module-cloud-multy-prerequisites: https://github.com/GlueOps/terraform-module-cloud-multy-prerequisites |
8990
| externalDns.aws_region | string | `"placeholder_aws_region"` | Should be the same `primary_region` you used in: https://github.com/GlueOps/terraform-module-cloud-multy-prerequisites |

‎TERRAFORM.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ module "glueops_platform_helm_values" {
88
dex_argocd_client_secret = "Zsbui/29YEqoGOzuI8snlqGcdaRYPSLocwLXDB5GhZY="
99
dex_grafana_client_secret = "AyYzghXw/qn/zfO6j9tN4H/7yLSYFPqnKOeoXOSi5U0="
1010
dex_vault_client_secret = "aLCZg513OvIA0vY5c24KLU2PrRXmBdhLGLUBrpkhBmE="
11-
dex_pomerium_client_secret = "5yon23Cwa83fscaq/CPTZ8UdhYIJ7gfHnl+gQO+FfPk="
1211
vault_aws_access_key = "AKIAU5Q3HAEIVOZFADIL"
1312
vault_aws_secret_key = "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"
1413
loki_aws_access_key = "AKIAA5QFHEEIFO6FYDE7"

‎main.tf‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -89,14 +89,18 @@ variable "dex_grafana_client_secret" {
8989
description = "Dex Grafana client secret"
9090
}
9191

92-
variable "dex_pomerium_client_secret" {
93-
description = "Dex Pomerium client secret"
94-
}
9592

9693
variable "dex_vault_client_secret" {
9794
description = "Dex Vault client secret"
9895
}
9996

97+
variable "dex_oauth2_client_secret" {
98+
description = "Dex Oauth2 client secret"
99+
}
100+
variable "dex_oauth2_cookie_secret" {
101+
description = "Dex Oauth2 cookie secret"
102+
}
103+
100104
variable "tenant_key" {
101105
type = string
102106
description = "this is also known as the tenant name or company key"
@@ -200,7 +204,7 @@ variable "vault_init_controller_aws_access_secret" {
200204

201205

202206
output "helm_values" {
203-
value = replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(
207+
value = replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(replace(
204208
replace(
205209
data.local_file.platform_values_template.content,
206210
"placeholder_tenant_key", var.tenant_key),
@@ -230,8 +234,9 @@ output "helm_values" {
230234
"placeholder_dex_github_client_secret", var.dex_github_client_secret),
231235
"placeholder_dex_argocd_client_secret", var.dex_argocd_client_secret),
232236
"placeholder_dex_grafana_client_secret", var.dex_grafana_client_secret),
233-
"placeholder_dex_pomerium_client_secret", var.dex_pomerium_client_secret),
234237
"placeholder_dex_vault_client_secret", var.dex_vault_client_secret),
238+
"placeholder_dex_oauth2_client_secret", var.dex_oauth2_client_secret),
239+
"placeholder_dex_oauth2_cookie_secret", var.dex_oauth2_cookie_secret),
235240
"placeholder_admin_github_org_name", var.admin_github_org_name),
236241
"placeholder_tenant_github_org_name", var.tenant_github_org_name),
237242
"placeholder_grafana_admin_password", var.grafana_admin_password),

‎templates/application-cluster-info-page.yaml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,12 @@ spec:
5252
5353
ingress:
5454
enabled: true
55-
ingressClassName: public-authenticated
55+
ingressClassName: glueops-platform
5656
annotations:
57-
ingress.pomerium.io/allow_any_authenticated_user: 'true'
57+
nginx.ingress.kubernetes.io/auth-signin: "http://oauth2.{{ .Values.captain_domain }}/oauth2/start?rd=$escaped_request_uri"
58+
nginx.ingress.kubernetes.io/auth-url: "http://oauth2.{{ .Values.captain_domain }}/oauth2/auth"
59+
nginx.ingress.kubernetes.io/auth-response-headers: "authorization, x-auth-request-user, x-auth-request-email, x_auth_request_access_token"
5860
entries:
59-
- name: public-authenticated
61+
- name: glueops-platform
6062
hosts:
6163
- hostname: cluster-info.{{ .Values.captain_domain }}

‎templates/application-dex.yaml‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,7 @@ spec:
3636
tag: {{ .Values.container_images.app_dex.dex.image.tag }}
3737
ingress:
3838
enabled: true
39-
className: "public-authenticated"
40-
annotations:
41-
ingress.pomerium.io/allow_public_unauthenticated_access: 'true'
39+
className: "glueops-platform"
4240
hosts:
4341
- host: dex.{{ .Values.captain_domain }}
4442
paths:
@@ -76,11 +74,11 @@ spec:
7674
redirectURIs:
7775
- 'https://grafana.{{ .Values.captain_domain }}/login/generic_oauth'
7876
secret: {{ .Values.dex.grafana.client_secret }}
79-
- id: pomerium
80-
name: Pomerium
77+
- id: oauth2-proxy
78+
name: OAuth2Proxy
8179
redirectURIs:
82-
- 'https://authenticated.{{ .Values.captain_domain }}/oauth2/callback'
83-
secret: {{ .Values.dex.pomerium.client_secret }}
80+
- 'https://oauth2.{{ .Values.captain_domain }}/oauth2/callback'
81+
secret: {{ .Values.dex.oauth2.client_secret }}
8482
- id: vault
8583
name: Hashicorp Vault
8684
redirectURIs:

‎templates/application-glueops-alerts.yaml‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -92,13 +92,13 @@ spec:
9292
rules:
9393
- alert: glueops-pod-in-bad-state
9494
expr: |
95-
(kube_pod_status_phase{namespace=~"^(kube-system|glueops-core-.*|glueops-core|pomerium|chisel-operator-system)$", pod!~"^(captain-redis-ha-configmap-test|captain-redis-ha-service-test)$", phase=~"^(Failed|Unknown|ContainerCreating|CrashLoopBackOff|ImagePullBackOff|ErrImageNeverPull|Pending)$"} == 1)
95+
(kube_pod_status_phase{namespace=~"^(kube-system|glueops-core-.*|glueops-core|chisel-operator-system)$", pod!~"^(captain-redis-ha-configmap-test|captain-redis-ha-service-test)$", phase=~"^(Failed|Unknown|ContainerCreating|CrashLoopBackOff|ImagePullBackOff|ErrImageNeverPull|Pending)$"} == 1)
9696
or
97-
(kube_pod_status_ready{namespace=~"^(kube-system|glueops-core-.*|glueops-core|pomerium|chisel-operator-system)$", pod!~"^(captain-redis-ha-configmap-test|captain-redis-ha-service-test|glueops-backup-and-exports-.*|.*-presync-.*|pomerium-gen-secrets-.*|argocd-redis-secret-.*)$", condition="false"} == 1)
97+
(kube_pod_status_ready{namespace=~"^(kube-system|glueops-core-.*|glueops-core|chisel-operator-system)$", pod!~"^(captain-redis-ha-configmap-test|captain-redis-ha-service-test|glueops-backup-and-exports-.*|.*-presync-.*|argocd-redis-secret-.*)$", condition="false"} == 1)
9898
or
99-
(kube_pod_status_restarts_total{namespace=~"^(kube-system|glueops-core-.*|glueops-core|pomerium|chisel-operator-system)$"} > 3)
99+
(kube_pod_status_restarts_total{namespace=~"^(kube-system|glueops-core-.*|glueops-core|chisel-operator-system)$"} > 3)
100100
or
101-
(kube_job_status_failed{namespace=~"^(kube-system|glueops-core-.*|glueops-core|pomerium|chisel-operator-system)$"} > 0)
101+
(kube_job_status_failed{namespace=~"^(kube-system|glueops-core-.*|glueops-core|chisel-operator-system)$"} > 0)
102102
for: 1m
103103
annotations:
104104
description: A GlueOps pod is in a bad state.
@@ -180,41 +180,41 @@ spec:
180180
apiVersion: metacontroller.glueops.dev/v1alpha1
181181
kind: LokiAlertRuleGroup
182182
metadata:
183-
name: glueops-services-high-5xx-rate-pomerium
183+
name: glueops-services-high-5xx-rate-nginx
184184
spec:
185-
name: glueops-services-high-5xx-rate-pomerium
185+
name: glueops-services-high-5xx-rate-nginx
186186
rules:
187-
- alert: glueops-services-high-5xx-rate-pomerium
187+
- alert: glueops-services-high-5xx-rate-nginx
188188
expr: |
189-
sum by (authority) (
190-
count_over_time({namespace="pomerium"} | json | response_code=~"5\\d{2}"[5m])
189+
sum(
190+
count_over_time({namespace="glueops-core-public-ingress-nginx"} | json | (status=~"5\\d{2}" or upstream_status=~"5\\d{2}")[5m])
191191
)
192192
/
193-
sum by (authority) (
194-
count_over_time({namespace="pomerium"} | json | response_code=~"\\d{3}"[5m])
193+
sum(
194+
count_over_time({namespace="glueops-core-public-ingress-nginx"} | json | response_code=~"\\d{3}"[5m])
195195
) > 0.01
196196
for: 1m
197197
annotations:
198-
description: GlueOps services are returning a high rate of 5xx responses via pomerium.
198+
description: GlueOps services are returning a high rate of 5xx responses via nginx.
199199
labels:
200200
alertname: glueops-core-alerts
201201
namespace: glueops-core-alerts
202202
- |-
203203
apiVersion: metacontroller.glueops.dev/v1alpha1
204204
kind: LokiAlertRuleGroup
205205
metadata:
206-
name: glueops-services-high-5xx-rate-nginx
206+
name: glueops-services-high-5xx-rate-glueops-platform-nginx
207207
spec:
208-
name: glueops-services-high-5xx-rate-nginx
208+
name: glueops-services-high-5xx-rate-glueops-platform-nginx
209209
rules:
210-
- alert: glueops-services-high-5xx-rate-nginx
210+
- alert: glueops-services-high-5xx-rate-glueops-platform-nginx
211211
expr: |
212212
sum(
213-
count_over_time({namespace="glueops-core-public-ingress-nginx"} | json | (status=~"5\\d{2}" or upstream_status=~"5\\d{2}")[5m])
213+
count_over_time({namespace="glueops-core-glueops-platform-ingress-nginx"} | json | (status=~"5\\d{2}" or upstream_status=~"5\\d{2}")[5m])
214214
)
215215
/
216216
sum(
217-
count_over_time({namespace="glueops-core-public-ingress-nginx"} | json | response_code=~"\\d{3}"[5m])
217+
count_over_time({namespace="glueops-core-glueops-platform-ingress-nginx"} | json | response_code=~"\\d{3}"[5m])
218218
) > 0.01
219219
for: 1m
220220
annotations:

‎templates/application-kube-prometheus-stack.yaml‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -103,13 +103,11 @@ spec:
103103
assertNoLeakedSecrets: false
104104
ingress:
105105
enabled: true
106-
ingressClassName: public-authenticated
106+
ingressClassName: glueops-platform
107107
annotations:
108-
ingress.pomerium.io/allow_any_authenticated_user: 'true'
109-
ingress.pomerium.io/pass_identity_headers: 'true'
110-
ingress.pomerium.io/allow_websockets: 'true'
111-
ingress.pomerium.io/idle_timeout: 0s
112-
ingress.pomerium.io/preserve_host_header: 'true'
108+
nginx.ingress.kubernetes.io/auth-signin: "http://oauth2.{{ .Values.captain_domain }}/oauth2/start?rd=$escaped_request_uri"
109+
nginx.ingress.kubernetes.io/auth-url: "http://oauth2.{{ .Values.captain_domain }}/oauth2/auth"
110+
nginx.ingress.kubernetes.io/auth-response-headers: "authorization, x-auth-request-user, x-auth-request-email, x_auth_request_access_token"
113111
hosts: ['grafana.{{ .Values.captain_domain }}']
114112
path: "/"
115113
additionalDataSources:

‎templates/application-nginx-glueops-platform-oauth2.yaml‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
apiVersion: argoproj.io/v1alpha1
2+
kind: Application
3+
metadata:
4+
name: glueops-platform-ingress-nginx
5+
annotations:
6+
argocd.argoproj.io/sync-wave: "0"
7+
finalizers:
8+
- resources-finalizer.argocd.argoproj.io
9+
spec:
10+
destination:
11+
name: "in-cluster"
12+
namespace: glueops-core-glueops-platform-ingress-nginx
13+
project: glueops-core
14+
syncPolicy:
15+
syncOptions:
16+
- CreateNamespace=true
17+
automated:
18+
prune: true
19+
selfHeal: true
20+
retry:
21+
backoff:
22+
duration: 10s
23+
factor: 2
24+
maxDuration: 3m0s
25+
limit: 5
26+
source:
27+
repoURL: 'https://kubernetes.github.io/ingress-nginx'
28+
chart: ingress-nginx
29+
targetRevision: 4.11.5
30+
helm:
31+
values: |-
32+
defaultBackend:
33+
image:
34+
registry: {{ .Values.base_registry }}
35+
controller:
36+
admissionWebhooks:
37+
patch:
38+
image:
39+
registry: {{ .Values.base_registry }}
40+
opentelemetry:
41+
image:
42+
registry: {{ .Values.base_registry }}
43+
image:
44+
registry: {{ .Values.base_registry }}
45+
image: {{ .Values.container_images.app_ingress_nginx.controller.image.repository }}
46+
tag: {{ .Values.container_images.app_ingress_nginx.controller.image.tag }}
47+
digest: ""
48+
admissionWebhooks:
49+
enabled: false
50+
replicaCount: {{ .Values.nginx.controller_replica_count }}
51+
maxUnavailable: 1
52+
config:
53+
use-forwarded-headers: true
54+
ssl-reject-handshake: true
55+
# https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/log-format.md
56+
# https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
57+
log-format-upstream: '{
58+
"time_iso8601": "$time_iso8601",
59+
"msec": "$msec",
60+
"remote_addr": "$remote_addr",
61+
"request": "$request",
62+
"proxy_protocol_addr": "$proxy_protocol_addr",
63+
"x_forwarded_for": "$proxy_add_x_forwarded_for",
64+
"req_id": "$req_id",
65+
"remote_user": "$remote_user",
66+
"bytes_sent": $bytes_sent,
67+
"request_time": $request_time,
68+
"status": $status,
69+
"body_bytes_sent": "$body_bytes_sent",
70+
"host": "$host",
71+
"server_protocol": "$server_protocol",
72+
"uri": "$uri",
73+
"args": "$args",
74+
"request_length": $request_length,
75+
"proxy_upstream_name": "$proxy_upstream_name",
76+
"proxy_alternative_upstream_name": "$proxy_alternative_upstream_name",
77+
"upstream_addr": "$upstream_addr",
78+
"upstream_response_length": "$upstream_response_length",
79+
"upstream_response_time": "$upstream_response_time",
80+
"upstream_status":"$upstream_status",
81+
"request_method": "$request_method",
82+
"http_referer": "$http_referer",
83+
"namespace": "$namespace",
84+
"ingress_name": "$ingress_name",
85+
"service_name": "$service_name",
86+
"service_port": "$service_port",
87+
"http_host": "$http_host",
88+
"scheme": "$scheme",
89+
"server_name": "$server_name",
90+
"upstream_cache_status": "$upstream_cache_status",
91+
"request_id": "$request_id",
92+
"ssl_protocol": "$ssl_protocol",
93+
"ssl_cipher": "$ssl_cipher",
94+
"http_user_agent": "$http_user_agent"
95+
}'
96+
extraArgs:
97+
default-ssl-certificate: glueops-core-cert-manager/{{ .Values.certManager.name_of_default_certificate }}
98+
service:
99+
annotations:
100+
service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
101+
external-dns.alpha.kubernetes.io/hostname: glueops-platform-ingress.{{ .Values.captain_domain }}
102+
type: "LoadBalancer"
103+
externalTrafficPolicy: "Local"
104+
# https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2366
105+
# https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2366#issuecomment-1788923154
106+
updateStrategy:
107+
rollingUpdate:
108+
maxSurge: 1
109+
maxUnavailable: 0
110+
# Add a pause to make time for the pod to be registered in the AWS NLB target group before proceeding with the next
111+
# https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/1834#issuecomment-781530724
112+
# https://alexklibisz.com/2021/07/20/speed-limits-for-rolling-restarts-in-kubernetes#round-3-set-minreadyseconds-maxunavailable-to-0-and-maxsurge-to-1
113+
minReadySeconds: 180
114+
# Add sleep on preStop to allow for graceful shutdown with AWS NLB
115+
# https://github.com/kubernetes/ingress-nginx/issues/6928
116+
# https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2366#issuecomment-1118312709
117+
lifecycle:
118+
preStop:
119+
exec:
120+
command: ["/bin/sh", "-c", "sleep 240; /wait-shutdown"]
121+
metrics:
122+
enabled: true
123+
serviceMonitor:
124+
enabled: true
125+
electionID: glueops-platform-ingress-nginx-leader
126+
ingressClassByName: true
127+
ingressClass: glueops-platform
128+
ingressClassResource:
129+
name: glueops-platform
130+
enabled: true
131+
default: false
132+
controllerValue: "k8s.io/glueops-platform-ingress-nginx"

0 commit comments

Comments
 (0)