From abd4ee798be3269beab14424c96c0c778289033c Mon Sep 17 00:00:00 2001 From: ION606 Date: Sat, 13 Sep 2025 10:14:48 -0400 Subject: [PATCH] attempt to add airflow ini 3 --- apps/0-project-and-root.yaml | 21 ++++++++++---- apps/children/airflow.yaml | 15 ++++++++-- apps/children/browser.yaml | 13 +++++++-- apps/children/coderunner.yaml | 13 +++++++-- apps/children/openwebui.yaml | 15 ++++++++-- apps/children/postgresql.yaml | 6 +++- apps/children/rag-server.yaml | 13 +++++++-- apps/children/searxng.yaml | 13 +++++++-- apps/children/tools.yaml | 13 +++++++-- apps/values/airflow.yaml | 52 +++++++++++++++++++++-------------- scripts/setup.sh | 18 ++++++------ temp.txt | 17 ++++++++++++ 12 files changed, 153 insertions(+), 56 deletions(-) create mode 100644 temp.txt diff --git a/apps/0-project-and-root.yaml b/apps/0-project-and-root.yaml index c66b928..af86767 100644 --- a/apps/0-project-and-root.yaml +++ b/apps/0-project-and-root.yaml @@ -1,6 +1,8 @@ apiVersion: argoproj.io/v1alpha1 kind: AppProject -metadata: { name: ai-stack, namespace: argocd } +metadata: + name: ai-stack + namespace: argocd spec: destinations: - server: https://kubernetes.default.svc @@ -13,14 +15,23 @@ spec: --- apiVersion: argoproj.io/v1alpha1 kind: Application -metadata: { name: ai-stack, namespace: argocd } +metadata: + name: ai-stack + namespace: argocd spec: project: ai-stack - destination: { server: https://kubernetes.default.svc, namespace: ai } + destination: + server: https://kubernetes.default.svc + namespace: ai source: repoURL: https://git.ion606.com/ion606/ollama-plus targetRevision: argo path: apps/children - directory: { recurse: true } + directory: + recurse: true syncPolicy: - automated: { prune: true, selfHeal: true } + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/apps/children/airflow.yaml b/apps/children/airflow.yaml index 7e86cdd..0324015 100644 --- a/apps/children/airflow.yaml +++ b/apps/children/airflow.yaml @@ -1,9 +1,15 @@ apiVersion: argoproj.io/v1alpha1 kind: Application -metadata: { name: airflow, namespace: ai } +metadata: + name: airflow + namespace: ai + annotations: + argocd.argoproj.io/sync-wave: "0" spec: project: ai-stack - destination: { server: https://kubernetes.default.svc, namespace: ai } + destination: + server: https://kubernetes.default.svc + namespace: ai sources: - repoURL: https://airflow.apache.org chart: airflow @@ -15,4 +21,7 @@ spec: - repoURL: https://git.ion606.com/ion606/ollama-plus targetRevision: argo ref: values - syncPolicy: { automated: { prune: true, selfHeal: true } } + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/apps/children/browser.yaml b/apps/children/browser.yaml index e8fd65b..435be84 100644 --- a/apps/children/browser.yaml +++ b/apps/children/browser.yaml @@ -1,11 +1,18 @@ apiVersion: argoproj.io/v1alpha1 kind: Application -metadata: { name: browser, namespace: ai } +metadata: + name: browser + namespace: ai spec: project: ai-stack - destination: { server: https://kubernetes.default.svc, namespace: ai } + destination: + server: https://kubernetes.default.svc + namespace: ai source: repoURL: https://your.git/repo.git targetRevision: main path: manifests/browser - syncPolicy: { automated: { prune: true, selfHeal: true } } + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/apps/children/coderunner.yaml b/apps/children/coderunner.yaml index b115c06..03cbfaf 100644 --- a/apps/children/coderunner.yaml +++ b/apps/children/coderunner.yaml @@ -1,11 +1,18 @@ apiVersion: argoproj.io/v1alpha1 kind: Application -metadata: { name: coderunner, namespace: ai } +metadata: + name: coderunner + namespace: ai spec: project: ai-stack - destination: { server: https://kubernetes.default.svc, namespace: ai } + destination: + server: https://kubernetes.default.svc + namespace: ai source: repoURL: https://git.ion606.com/ion606/ollama-plus targetRevision: main path: manifests/coderunner - syncPolicy: { automated: { prune: true, selfHeal: true } } + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/apps/children/openwebui.yaml b/apps/children/openwebui.yaml index f3ea013..5259821 100644 --- a/apps/children/openwebui.yaml +++ b/apps/children/openwebui.yaml @@ -1,9 +1,15 @@ apiVersion: argoproj.io/v1alpha1 kind: Application -metadata: { name: openwebui, namespace: ai } +metadata: + name: openwebui + namespace: ai + annotations: + argocd.argoproj.io/sync-wave: "0" spec: project: ai-stack - destination: { server: https://kubernetes.default.svc, namespace: ai } + destination: + server: https://kubernetes.default.svc + namespace: ai source: repoURL: https://helm.openwebui.com chart: open-webui @@ -11,4 +17,7 @@ spec: helm: valueFiles: - apps/values/openwebui.yaml - syncPolicy: { automated: { prune: true, selfHeal: true } } + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/apps/children/postgresql.yaml b/apps/children/postgresql.yaml index 795a56d..e3bbfbe 100644 --- a/apps/children/postgresql.yaml +++ b/apps/children/postgresql.yaml @@ -1,6 +1,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Application -metadata: { name: postgresql, namespace: ai } +metadata: + name: postgresql + namespace: ai + annotations: + argocd.argoproj.io/sync-wave: "-10" spec: project: ai-stack destination: diff --git a/apps/children/rag-server.yaml b/apps/children/rag-server.yaml index 3ac4a67..0b5eb66 100644 --- a/apps/children/rag-server.yaml +++ b/apps/children/rag-server.yaml @@ -1,11 +1,18 @@ apiVersion: argoproj.io/v1alpha1 kind: Application -metadata: { name: rag-server, namespace: ai } +metadata: + name: rag-server + namespace: ai spec: project: ai-stack - destination: { server: https://kubernetes.default.svc, namespace: ai } + destination: + server: https://kubernetes.default.svc + namespace: ai source: repoURL: https://git.ion606.com/ion606/ollama-plus targetRevision: main path: manifests/rag-server - syncPolicy: { automated: { prune: true, selfHeal: true } } + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/apps/children/searxng.yaml b/apps/children/searxng.yaml index 6fb15e8..e501c8c 100644 --- a/apps/children/searxng.yaml +++ b/apps/children/searxng.yaml @@ -1,9 +1,13 @@ apiVersion: argoproj.io/v1alpha1 kind: Application -metadata: { name: searxng, namespace: ai } +metadata: + name: searxng + namespace: ai spec: project: ai-stack - destination: { server: https://kubernetes.default.svc, namespace: ai } + destination: + server: https://kubernetes.default.svc + namespace: ai source: repoURL: https://charts.kubito.dev chart: searxng @@ -11,4 +15,7 @@ spec: helm: valueFiles: - apps/values/searxng.yaml - syncPolicy: { automated: { prune: true, selfHeal: true } } + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/apps/children/tools.yaml b/apps/children/tools.yaml index 162d2a7..13bf5a1 100644 --- a/apps/children/tools.yaml +++ b/apps/children/tools.yaml @@ -1,11 +1,18 @@ apiVersion: argoproj.io/v1alpha1 kind: Application -metadata: { name: tools, namespace: ai } +metadata: + name: tools + namespace: ai spec: project: ai-stack - destination: { server: https://kubernetes.default.svc, namespace: ai } + destination: + server: https://kubernetes.default.svc + namespace: ai source: repoURL: https://git.ion606.com/ion606/ollama-plus targetRevision: main path: manifests/tools - syncPolicy: { automated: { prune: true, selfHeal: true } } + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/apps/values/airflow.yaml b/apps/values/airflow.yaml index 8fde7ee..8e50f5c 100644 --- a/apps/values/airflow.yaml +++ b/apps/values/airflow.yaml @@ -11,26 +11,26 @@ env: # using external postgres and not the subchart postgresql: - enabled: false; + enabled: false data: metadataConnection: - user: postgres; - pass: mypassword; - protocol: postgresql; - host: postgresql-primary.ai.svc.cluster.local; - port: 5432; - db: airflow; # better to keep airflow separate from other apps - sslmode: disable; + user: postgres + pass: mypassword + protocol: postgresql + host: postgresql-primary.ai.svc.cluster.local + port: 5432 + db: airflow # better to keep airflow separate from other apps + sslmode: disable # enable pgbouncer and sync with extraIni pgbouncer: - enabled: true; + enabled: true # official default is scram-sha-256, but bitnami uses md5 so... - auth_type: md5; - maxClientConn: 200; - metadataPoolSize: 20; # server connections to metadata db - resultBackendPoolSize: 5; + auth_type: md5 + maxClientConn: 200 + metadataPoolSize: 20 # server connections to metadata db + resultBackendPoolSize: 5 # general pgbouncer section stuffs extraIni: | pool_mode = transaction @@ -39,7 +39,7 @@ pgbouncer: ignore_startup_parameters = extra_float_digits server_tls_sslmode = prefer metricsExporterSidecar: - enabled: true; + enabled: true logs: persistence: @@ -50,8 +50,12 @@ allowPodLaunching: true scheduler: resources: - requests: { cpu: "200m", memory: "512Mi" } - limits: { cpu: "1", memory: "1Gi" } + requests: + cpu: "200m" + memory: "512Mi" + limits: + cpu: "1" + memory: "1Gi" webserver: webserverSecretKeySecretName: airflow-webserver-secret @@ -59,13 +63,21 @@ webserver: type: NodePort nodePort: 30082 # 30000–32767 resources: - requests: { cpu: "100m", memory: "256Mi" } - limits: { cpu: "500m", memory: "512Mi" } + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" triggerer: resources: - requests: { cpu: "50m", memory: "128Mi" } - limits: { cpu: "200m", memory: "256Mi" } + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "200m" + memory: "256Mi" # bc using nodeport ingress: diff --git a/scripts/setup.sh b/scripts/setup.sh index 889eb62..bfd069c 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -32,18 +32,18 @@ kubectl apply -n argocd -f apps/0-project-and-root.yaml; # kubectl -n ai create secret generic airflow-webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets;print(secrets.token_hex(16))')"; # airflow stuffs -kubectl -n ai exec -it svc/postgresql-primary -- bash -lc "psql -U postgres -c 'CREATE DATABASE airflow;'"; -set -l PGBOUNCER_SVC (kubectl -n ai get svc -l tier=airflow,component=pgbouncer -o jsonpath='{.items[0].metadata.name}'); -echo $PGBOUNCER_SVC; +# kubectl -n ai exec -it svc/postgresql-primary -- bash -lc "psql -U postgres -c 'CREATE DATABASE airflow;'"; +# set -l PGBOUNCER_SVC (kubectl -n ai get svc -l tier=airflow,component=pgbouncer -o jsonpath='{.items[0].metadata.name}'); +# echo $PGBOUNCER_SVC; -# stats (used by the metrics sidecar and also points to pgbouncer itself on 127.0.0.1:6543) -kubectl -n ai create secret generic airflow-pgbouncer-stats \ - --from-literal=connection="postgresql://postgres:mypassword@127.0.0.1:6543/pgbouncer?sslmode=disable"; +# # stats (used by the metrics sidecar and also points to pgbouncer itself on 127.0.0.1:6543) +# kubectl -n ai create secret generic airflow-pgbouncer-stats \ +# --from-literal=connection="postgresql://postgres:mypassword@127.0.0.1:6543/pgbouncer?sslmode=disable"; -kubectl -n ai create secret generic airflow-metadata \ - --from-literal=connection="postgresql+psycopg2://postgres:mypassword@$PGBOUNCER_SVC:6543/airflow"; +# kubectl -n ai create secret generic airflow-metadata \ +# --from-literal=connection="postgresql+psycopg2://postgres:mypassword@$PGBOUNCER_SVC:6543/airflow"; -minikube service -n ai airflow-webserver --url || echo "FAILED TO FIND SERVICE" +# minikube service -n ai airflow-webserver --url || echo "FAILED TO FIND SERVICE" # port-forward argocd ui echo ""; diff --git a/temp.txt b/temp.txt new file mode 100644 index 0000000..70c89b2 --- /dev/null +++ b/temp.txt @@ -0,0 +1,17 @@ +|---------------|-----------------------------------------|--------------|---------------------------| +| NAMESPACE | NAME | TARGET PORT | URL | +|---------------|-----------------------------------------|--------------|---------------------------| +| argocd | argocd-applicationset-controller | No node port | | +| argocd | argocd-dex-server | No node port | | +| argocd | argocd-metrics | No node port | | +| argocd | argocd-notifications-controller-metrics | No node port | | +| argocd | argocd-redis | No node port | | +| argocd | argocd-repo-server | No node port | | +| argocd | argocd-server | No node port | | +| argocd | argocd-server-metrics | No node port | | +| default | kubernetes | No node port | | +| ingress-nginx | ingress-nginx-controller | http/80 | http://192.168.49.2:31684 | +| | | https/443 | http://192.168.49.2:31486 | +| ingress-nginx | ingress-nginx-controller-admission | No node port | | +| kube-system | kube-dns | No node port | | +|---------------|-----------------------------------------|--------------|---------------------------|