I gave up on airflow
This commit is contained in:
@@ -134,3 +134,4 @@ __pycache__/
|
|||||||
.venv/
|
.venv/
|
||||||
|
|
||||||
*.xml
|
*.xml
|
||||||
|
temp.*
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
apiVersion: argoproj.io/v1alpha1
|
|
||||||
kind: Application
|
|
||||||
metadata:
|
|
||||||
name: airflow
|
|
||||||
namespace: ai
|
|
||||||
annotations:
|
|
||||||
argocd.argoproj.io/sync-wave: "0"
|
|
||||||
spec:
|
|
||||||
project: ai-stack
|
|
||||||
destination:
|
|
||||||
server: https://kubernetes.default.svc
|
|
||||||
namespace: ai
|
|
||||||
sources:
|
|
||||||
- repoURL: https://airflow.apache.org
|
|
||||||
chart: airflow
|
|
||||||
targetRevision: "*"
|
|
||||||
helm:
|
|
||||||
valueFiles:
|
|
||||||
- $values/values/airflow.yaml
|
|
||||||
|
|
||||||
- repoURL: https://git.ion606.com/ion606/ollama-plus
|
|
||||||
targetRevision: argo
|
|
||||||
ref: values
|
|
||||||
syncPolicy:
|
|
||||||
automated:
|
|
||||||
prune: true
|
|
||||||
selfHeal: true
|
|
||||||
@@ -1,90 +0,0 @@
|
|||||||
useStandardNaming: true
|
|
||||||
executor: KubernetesExecutor
|
|
||||||
|
|
||||||
airflow:
|
|
||||||
extraPipPackages: []
|
|
||||||
|
|
||||||
# metastore (postgres)
|
|
||||||
env:
|
|
||||||
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
|
||||||
value: "postgresql+psycopg2://postgres:mypassword@postgresql-primary.ai.svc.cluster.local:5432/airflow"
|
|
||||||
|
|
||||||
# using external postgres and not the subchart
|
|
||||||
postgresql:
|
|
||||||
enabled: false
|
|
||||||
|
|
||||||
data:
|
|
||||||
metadataConnection:
|
|
||||||
user: postgres
|
|
||||||
pass: mypassword
|
|
||||||
protocol: postgresql
|
|
||||||
host: postgresql-primary.ai.svc.cluster.local
|
|
||||||
port: 5432
|
|
||||||
db: airflow # better to keep airflow separate from other apps
|
|
||||||
sslmode: disable
|
|
||||||
|
|
||||||
# enable pgbouncer and sync with extraIni
|
|
||||||
pgbouncer:
|
|
||||||
enabled: true
|
|
||||||
# official default is scram-sha-256, but bitnami uses md5 so...
|
|
||||||
auth_type: md5
|
|
||||||
maxClientConn: 200
|
|
||||||
metadataPoolSize: 20 # server connections to metadata db
|
|
||||||
resultBackendPoolSize: 5
|
|
||||||
# general pgbouncer section stuffs
|
|
||||||
extraIni: |
|
|
||||||
pool_mode = transaction
|
|
||||||
listen_port = 6543
|
|
||||||
listen_addr = *
|
|
||||||
ignore_startup_parameters = extra_float_digits
|
|
||||||
server_tls_sslmode = prefer
|
|
||||||
metricsExporterSidecar:
|
|
||||||
enabled: true
|
|
||||||
|
|
||||||
logs:
|
|
||||||
persistence:
|
|
||||||
enabled: true
|
|
||||||
size: 2Gi
|
|
||||||
|
|
||||||
allowPodLaunching: true
|
|
||||||
|
|
||||||
scheduler:
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: "200m"
|
|
||||||
memory: "512Mi"
|
|
||||||
limits:
|
|
||||||
cpu: "1"
|
|
||||||
memory: "1Gi"
|
|
||||||
|
|
||||||
webserver:
|
|
||||||
webserverSecretKeySecretName: airflow-webserver-secret
|
|
||||||
service:
|
|
||||||
type: NodePort
|
|
||||||
nodePort: 30082 # 30000–32767
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: "100m"
|
|
||||||
memory: "256Mi"
|
|
||||||
limits:
|
|
||||||
cpu: "500m"
|
|
||||||
memory: "512Mi"
|
|
||||||
|
|
||||||
triggerer:
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: "50m"
|
|
||||||
memory: "128Mi"
|
|
||||||
limits:
|
|
||||||
cpu: "200m"
|
|
||||||
memory: "256Mi"
|
|
||||||
|
|
||||||
# bc using nodeport
|
|
||||||
ingress:
|
|
||||||
enabled: false
|
|
||||||
|
|
||||||
# naur helm hooks for these jobs
|
|
||||||
createUserJob:
|
|
||||||
useHelmHooks: false
|
|
||||||
migrateDatabaseJob:
|
|
||||||
useHelmHooks: false
|
|
||||||
@@ -26,25 +26,6 @@ kubectl rollout status deploy/argocd-application-controller -n argocd --timeout=
|
|||||||
# NOTE: creates the child Applications in apps/children/*
|
# NOTE: creates the child Applications in apps/children/*
|
||||||
kubectl apply -n argocd -f apps/0-project-and-root.yaml;
|
kubectl apply -n argocd -f apps/0-project-and-root.yaml;
|
||||||
|
|
||||||
# service!
|
|
||||||
# SEE???? I CAN USE DASHES AND NOT JUST CAMELCASE!!!
|
|
||||||
# kubectl -n ai create secret generic airflow-fernet-key-secret --from-literal=fernet-key=$(python3 -c 'import secrets;print(secrets.token_urlsafe(32))')
|
|
||||||
# kubectl -n ai create secret generic airflow-webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets;print(secrets.token_hex(16))')";
|
|
||||||
|
|
||||||
# airflow stuffs
|
|
||||||
# kubectl -n ai exec -it svc/postgresql-primary -- bash -lc "psql -U postgres -c 'CREATE DATABASE airflow;'";
|
|
||||||
# set -l PGBOUNCER_SVC (kubectl -n ai get svc -l tier=airflow,component=pgbouncer -o jsonpath='{.items[0].metadata.name}');
|
|
||||||
# echo $PGBOUNCER_SVC;
|
|
||||||
|
|
||||||
# # stats (used by the metrics sidecar and also points to pgbouncer itself on 127.0.0.1:6543)
|
|
||||||
# kubectl -n ai create secret generic airflow-pgbouncer-stats \
|
|
||||||
# --from-literal=connection="postgresql://postgres:mypassword@127.0.0.1:6543/pgbouncer?sslmode=disable";
|
|
||||||
|
|
||||||
# kubectl -n ai create secret generic airflow-metadata \
|
|
||||||
# --from-literal=connection="postgresql+psycopg2://postgres:mypassword@$PGBOUNCER_SVC:6543/airflow";
|
|
||||||
|
|
||||||
# minikube service -n ai airflow-webserver --url || echo "FAILED TO FIND SERVICE"
|
|
||||||
|
|
||||||
# port-forward argocd ui
|
# port-forward argocd ui
|
||||||
echo "";
|
echo "";
|
||||||
echo "argocd initial admin password (username 'admin'):";
|
echo "argocd initial admin password (username 'admin'):";
|
||||||
|
|||||||
@@ -1,17 +0,0 @@
|
|||||||
|---------------|-----------------------------------------|--------------|---------------------------|
|
|
||||||
| NAMESPACE | NAME | TARGET PORT | URL |
|
|
||||||
|---------------|-----------------------------------------|--------------|---------------------------|
|
|
||||||
| argocd | argocd-applicationset-controller | No node port | |
|
|
||||||
| argocd | argocd-dex-server | No node port | |
|
|
||||||
| argocd | argocd-metrics | No node port | |
|
|
||||||
| argocd | argocd-notifications-controller-metrics | No node port | |
|
|
||||||
| argocd | argocd-redis | No node port | |
|
|
||||||
| argocd | argocd-repo-server | No node port | |
|
|
||||||
| argocd | argocd-server | No node port | |
|
|
||||||
| argocd | argocd-server-metrics | No node port | |
|
|
||||||
| default | kubernetes | No node port | |
|
|
||||||
| ingress-nginx | ingress-nginx-controller | http/80 | http://192.168.49.2:31684 |
|
|
||||||
| | | https/443 | http://192.168.49.2:31486 |
|
|
||||||
| ingress-nginx | ingress-nginx-controller-admission | No node port | |
|
|
||||||
| kube-system | kube-dns | No node port | |
|
|
||||||
|---------------|-----------------------------------------|--------------|---------------------------|
|
|
||||||
Reference in New Issue
Block a user