I gave up on airflow
This commit is contained in:
@@ -134,3 +134,4 @@ __pycache__/
|
||||
.venv/
|
||||
|
||||
*.xml
|
||||
temp.*
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: airflow
|
||||
namespace: ai
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "0"
|
||||
spec:
|
||||
project: ai-stack
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: ai
|
||||
sources:
|
||||
- repoURL: https://airflow.apache.org
|
||||
chart: airflow
|
||||
targetRevision: "*"
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/values/airflow.yaml
|
||||
|
||||
- repoURL: https://git.ion606.com/ion606/ollama-plus
|
||||
targetRevision: argo
|
||||
ref: values
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
@@ -1,90 +0,0 @@
|
||||
useStandardNaming: true
|
||||
executor: KubernetesExecutor
|
||||
|
||||
airflow:
|
||||
extraPipPackages: []
|
||||
|
||||
# metastore (postgres)
|
||||
env:
|
||||
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
||||
value: "postgresql+psycopg2://postgres:mypassword@postgresql-primary.ai.svc.cluster.local:5432/airflow"
|
||||
|
||||
# using external postgres and not the subchart
|
||||
postgresql:
|
||||
enabled: false
|
||||
|
||||
data:
|
||||
metadataConnection:
|
||||
user: postgres
|
||||
pass: mypassword
|
||||
protocol: postgresql
|
||||
host: postgresql-primary.ai.svc.cluster.local
|
||||
port: 5432
|
||||
db: airflow # better to keep airflow separate from other apps
|
||||
sslmode: disable
|
||||
|
||||
# enable pgbouncer and sync with extraIni
|
||||
pgbouncer:
|
||||
enabled: true
|
||||
# official default is scram-sha-256, but bitnami uses md5 so...
|
||||
auth_type: md5
|
||||
maxClientConn: 200
|
||||
metadataPoolSize: 20 # server connections to metadata db
|
||||
resultBackendPoolSize: 5
|
||||
# general pgbouncer section stuffs
|
||||
extraIni: |
|
||||
pool_mode = transaction
|
||||
listen_port = 6543
|
||||
listen_addr = *
|
||||
ignore_startup_parameters = extra_float_digits
|
||||
server_tls_sslmode = prefer
|
||||
metricsExporterSidecar:
|
||||
enabled: true
|
||||
|
||||
logs:
|
||||
persistence:
|
||||
enabled: true
|
||||
size: 2Gi
|
||||
|
||||
allowPodLaunching: true
|
||||
|
||||
scheduler:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "200m"
|
||||
memory: "512Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: "1Gi"
|
||||
|
||||
webserver:
|
||||
webserverSecretKeySecretName: airflow-webserver-secret
|
||||
service:
|
||||
type: NodePort
|
||||
nodePort: 30082 # 30000–32767
|
||||
resources:
|
||||
requests:
|
||||
cpu: "100m"
|
||||
memory: "256Mi"
|
||||
limits:
|
||||
cpu: "500m"
|
||||
memory: "512Mi"
|
||||
|
||||
triggerer:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "50m"
|
||||
memory: "128Mi"
|
||||
limits:
|
||||
cpu: "200m"
|
||||
memory: "256Mi"
|
||||
|
||||
# bc using nodeport
|
||||
ingress:
|
||||
enabled: false
|
||||
|
||||
# naur helm hooks for these jobs
|
||||
createUserJob:
|
||||
useHelmHooks: false
|
||||
migrateDatabaseJob:
|
||||
useHelmHooks: false
|
||||
@@ -26,25 +26,6 @@ kubectl rollout status deploy/argocd-application-controller -n argocd --timeout=
|
||||
# NOTE: creates the child Applications in apps/children/*
|
||||
kubectl apply -n argocd -f apps/0-project-and-root.yaml;
|
||||
|
||||
# service!
|
||||
# SEE???? I CAN USE DASHES AND NOT JUST CAMELCASE!!!
|
||||
# kubectl -n ai create secret generic airflow-fernet-key-secret --from-literal=fernet-key=$(python3 -c 'import secrets;print(secrets.token_urlsafe(32))')
|
||||
# kubectl -n ai create secret generic airflow-webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets;print(secrets.token_hex(16))')";
|
||||
|
||||
# airflow stuffs
|
||||
# kubectl -n ai exec -it svc/postgresql-primary -- bash -lc "psql -U postgres -c 'CREATE DATABASE airflow;'";
|
||||
# set -l PGBOUNCER_SVC (kubectl -n ai get svc -l tier=airflow,component=pgbouncer -o jsonpath='{.items[0].metadata.name}');
|
||||
# echo $PGBOUNCER_SVC;
|
||||
|
||||
# # stats (used by the metrics sidecar and also points to pgbouncer itself on 127.0.0.1:6543)
|
||||
# kubectl -n ai create secret generic airflow-pgbouncer-stats \
|
||||
# --from-literal=connection="postgresql://postgres:mypassword@127.0.0.1:6543/pgbouncer?sslmode=disable";
|
||||
|
||||
# kubectl -n ai create secret generic airflow-metadata \
|
||||
# --from-literal=connection="postgresql+psycopg2://postgres:mypassword@$PGBOUNCER_SVC:6543/airflow";
|
||||
|
||||
# minikube service -n ai airflow-webserver --url || echo "FAILED TO FIND SERVICE"
|
||||
|
||||
# port-forward argocd ui
|
||||
echo "";
|
||||
echo "argocd initial admin password (username 'admin'):";
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
|---------------|-----------------------------------------|--------------|---------------------------|
|
||||
| NAMESPACE | NAME | TARGET PORT | URL |
|
||||
|---------------|-----------------------------------------|--------------|---------------------------|
|
||||
| argocd | argocd-applicationset-controller | No node port | |
|
||||
| argocd | argocd-dex-server | No node port | |
|
||||
| argocd | argocd-metrics | No node port | |
|
||||
| argocd | argocd-notifications-controller-metrics | No node port | |
|
||||
| argocd | argocd-redis | No node port | |
|
||||
| argocd | argocd-repo-server | No node port | |
|
||||
| argocd | argocd-server | No node port | |
|
||||
| argocd | argocd-server-metrics | No node port | |
|
||||
| default | kubernetes | No node port | |
|
||||
| ingress-nginx | ingress-nginx-controller | http/80 | http://192.168.49.2:31684 |
|
||||
| | | https/443 | http://192.168.49.2:31486 |
|
||||
| ingress-nginx | ingress-nginx-controller-admission | No node port | |
|
||||
| kube-system | kube-dns | No node port | |
|
||||
|---------------|-----------------------------------------|--------------|---------------------------|
|
||||
Reference in New Issue
Block a user