added airflow pipe

This commit is contained in:
2025-09-12 21:52:11 -04:00
parent d07892566f
commit ddc95882ee
4 changed files with 78 additions and 3 deletions
+15
View File
@@ -0,0 +1,15 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata: { name: airflow, namespace: argocd }
spec:
project: ai-stack
destination: { server: https://kubernetes.default.svc, namespace: ai }
source:
repoURL: https://airflow.apache.org
chart: airflow
targetRevision: "*"
helm:
valueFiles:
- apps/values/airflow.yaml
syncPolicy:
automated: { prune: true, selfHeal: true }
+53
View File
@@ -0,0 +1,53 @@
useStandardNaming: true
executor: KubernetesExecutor
# install k8s provider for kubernetesexecutor
airflow:
extraPipPackages:
- "apache-airflow-providers-cncf-kubernetes>=7.4.0"
# use your existing postgres for the metastore
env:
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
value: "postgresql+psycopg2://postgres:mypassword@postgresql-primary.ai.svc.cluster.local:5432/openwebui"
pgbouncer:
enabled: true
logs:
persistence:
enabled: true
size: 2Gi
allowPodLaunching: true
# small footprints for minikube
scheduler:
resources:
requests: { cpu: "200m", memory: "512Mi" }
limits: { cpu: "1", memory: "1Gi" }
webserver:
secretKeySecretName: airflow-webserver-secret
fernetKeySecretName: airflow-fernet-key-secret
service:
type: NodePort
nodePort: 30082 # 3000032767
resources:
requests: { cpu: "100m", memory: "256Mi" }
limits: { cpu: "500m", memory: "512Mi" }
triggerer:
resources:
requests: { cpu: "50m", memory: "128Mi" }
limits: { cpu: "200m", memory: "256Mi" }
# bc using nodeport
ingress:
enabled: false
# naur helm hooks for these jobs
createUserJob:
useHelmHooks: false
migrateDatabaseJob:
useHelmHooks: false
+3 -3
View File
@@ -4,7 +4,7 @@ image:
pullPolicy: IfNotPresent
service:
type: NodePort # or ClusterIP if youll use ingress below
type: NodePort # or ClusterIP if actually using ingress below
nodePort: 4000
persistence:
@@ -12,7 +12,7 @@ persistence:
size: 5Gi
ingress:
enabled: false # set true fo http://openwebui.local via nginx
enabled: false # set true for http://openwebui.local via nginx
className: nginx
hosts:
- host: openwebui.local
@@ -35,6 +35,6 @@ extraEnvVars:
value: "postgresql://postgres:mypassword@postgresql-primary.ai.svc.cluster.local:5432/openwebui"
- name: PGVECTOR_CREATE_EXTENSION
value: "true"
# set canonical url for oauth/web search callbacks, etc.
# set canonical url for oauth/web search callbacks, etc
- name: WEBUI_URL
value: "http://openwebui.local"
+7
View File
@@ -26,6 +26,13 @@ kubectl rollout status deploy/argocd-application-controller -n argocd --timeout=
# NOTE: creates the child Applications in apps/children/*
kubectl apply -n argocd -f apps/0-project-and-root.yaml;
# service!
# SEE???? I CAN USE DASHES AND NOT JUST CAMELCASE!!!
kubectl -n ai create secret generic airflow-fernet-key-secret --from-literal=fernet-key=$(python3 -c 'import secrets;print(secrets.token_urlsafe(32))')
kubectl -n ai create secret generic airflow-webserver-secret --from-literal=webserver-secret-key=$(python3 -c 'import secrets;print(secrets.token_hex(16))')
minikube service -n ai airflow-webserver --url
# port-forward argocd ui
echo "";
echo "argocd initial admin password (username 'admin'):";