From 469dfcd094074a88bf2d1206236ba7ad0efa75e7 Mon Sep 17 00:00:00 2001 From: ION606 Date: Sat, 13 Sep 2025 10:23:14 -0400 Subject: [PATCH] I gave up on airflow --- .gitignore | 1 + apps/children/airflow.yaml | 27 ------------ apps/values/airflow.yaml | 90 -------------------------------------- scripts/setup.sh | 19 -------- temp.txt | 17 ------- 5 files changed, 1 insertion(+), 153 deletions(-) delete mode 100644 apps/children/airflow.yaml delete mode 100644 apps/values/airflow.yaml delete mode 100644 temp.txt diff --git a/.gitignore b/.gitignore index 9041e60..b02f485 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,4 @@ __pycache__/ .venv/ *.xml +temp.* diff --git a/apps/children/airflow.yaml b/apps/children/airflow.yaml deleted file mode 100644 index 0324015..0000000 --- a/apps/children/airflow.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: airflow - namespace: ai - annotations: - argocd.argoproj.io/sync-wave: "0" -spec: - project: ai-stack - destination: - server: https://kubernetes.default.svc - namespace: ai - sources: - - repoURL: https://airflow.apache.org - chart: airflow - targetRevision: "*" - helm: - valueFiles: - - $values/values/airflow.yaml - - - repoURL: https://git.ion606.com/ion606/ollama-plus - targetRevision: argo - ref: values - syncPolicy: - automated: - prune: true - selfHeal: true diff --git a/apps/values/airflow.yaml b/apps/values/airflow.yaml deleted file mode 100644 index 8e50f5c..0000000 --- a/apps/values/airflow.yaml +++ /dev/null @@ -1,90 +0,0 @@ -useStandardNaming: true -executor: KubernetesExecutor - -airflow: - extraPipPackages: [] - -# metastore (postgres) -env: - - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN - value: "postgresql+psycopg2://postgres:mypassword@postgresql-primary.ai.svc.cluster.local:5432/airflow" - -# using external postgres and not the subchart -postgresql: - enabled: false - -data: - metadataConnection: - user: postgres - pass: mypassword - protocol: postgresql - host: postgresql-primary.ai.svc.cluster.local - port: 5432 - db: airflow # better to keep airflow separate from other apps - sslmode: disable - -# enable pgbouncer and sync with extraIni -pgbouncer: - enabled: true - # official default is scram-sha-256, but bitnami uses md5 so... - auth_type: md5 - maxClientConn: 200 - metadataPoolSize: 20 # server connections to metadata db - resultBackendPoolSize: 5 - # general pgbouncer section stuffs - extraIni: | - pool_mode = transaction - listen_port = 6543 - listen_addr = * - ignore_startup_parameters = extra_float_digits - server_tls_sslmode = prefer - metricsExporterSidecar: - enabled: true - -logs: - persistence: - enabled: true - size: 2Gi - -allowPodLaunching: true - -scheduler: - resources: - requests: - cpu: "200m" - memory: "512Mi" - limits: - cpu: "1" - memory: "1Gi" - -webserver: - webserverSecretKeySecretName: airflow-webserver-secret - service: - type: NodePort - nodePort: 30082 # 30000–32767 - resources: - requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "500m" - memory: "512Mi" - -triggerer: - resources: - requests: - cpu: "50m" - memory: "128Mi" - limits: - cpu: "200m" - memory: "256Mi" - -# bc using nodeport -ingress: - enabled: false - -# naur helm hooks for these jobs -createUserJob: - useHelmHooks: false -migrateDatabaseJob: - useHelmHooks: false diff --git a/scripts/setup.sh b/scripts/setup.sh index bfd069c..1f6c7e8 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -26,25 +26,6 @@ kubectl rollout status deploy/argocd-application-controller -n argocd --timeout= # NOTE: creates the child Applications in apps/children/* kubectl apply -n argocd -f apps/0-project-and-root.yaml; -# service! -# SEE???? I CAN USE DASHES AND NOT JUST CAMELCASE!!! -# kubectl -n ai create secret generic airflow-fernet-key-secret --from-literal=fernet-key=$(python3 -c 'import secrets;print(secrets.token_urlsafe(32))') -# kubectl -n ai create secret generic airflow-webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets;print(secrets.token_hex(16))')"; - -# airflow stuffs -# kubectl -n ai exec -it svc/postgresql-primary -- bash -lc "psql -U postgres -c 'CREATE DATABASE airflow;'"; -# set -l PGBOUNCER_SVC (kubectl -n ai get svc -l tier=airflow,component=pgbouncer -o jsonpath='{.items[0].metadata.name}'); -# echo $PGBOUNCER_SVC; - -# # stats (used by the metrics sidecar and also points to pgbouncer itself on 127.0.0.1:6543) -# kubectl -n ai create secret generic airflow-pgbouncer-stats \ -# --from-literal=connection="postgresql://postgres:mypassword@127.0.0.1:6543/pgbouncer?sslmode=disable"; - -# kubectl -n ai create secret generic airflow-metadata \ -# --from-literal=connection="postgresql+psycopg2://postgres:mypassword@$PGBOUNCER_SVC:6543/airflow"; - -# minikube service -n ai airflow-webserver --url || echo "FAILED TO FIND SERVICE" - # port-forward argocd ui echo ""; echo "argocd initial admin password (username 'admin'):"; diff --git a/temp.txt b/temp.txt deleted file mode 100644 index 70c89b2..0000000 --- a/temp.txt +++ /dev/null @@ -1,17 +0,0 @@ -|---------------|-----------------------------------------|--------------|---------------------------| -| NAMESPACE | NAME | TARGET PORT | URL | -|---------------|-----------------------------------------|--------------|---------------------------| -| argocd | argocd-applicationset-controller | No node port | | -| argocd | argocd-dex-server | No node port | | -| argocd | argocd-metrics | No node port | | -| argocd | argocd-notifications-controller-metrics | No node port | | -| argocd | argocd-redis | No node port | | -| argocd | argocd-repo-server | No node port | | -| argocd | argocd-server | No node port | | -| argocd | argocd-server-metrics | No node port | | -| default | kubernetes | No node port | | -| ingress-nginx | ingress-nginx-controller | http/80 | http://192.168.49.2:31684 | -| | | https/443 | http://192.168.49.2:31486 | -| ingress-nginx | ingress-nginx-controller-admission | No node port | | -| kube-system | kube-dns | No node port | | -|---------------|-----------------------------------------|--------------|---------------------------|