bug: stuck

Taskfile.yml

@@ -6,6 +6,7 @@ tasks:
   tf:destroy: ansible-playbook playbooks/destroy.yml {{.CLI_ARGS}}
   configure-nat: ansible-playbook playbooks/configure_nat.yml {{.CLI_ARGS}}
   configure-servers: ansible-playbook playbooks/configure_servers.yml {{.CLI_ARGS}}
+  deploy: ansible-playbook playbooks/install_k8s.yml {{.CLI_ARGS}}
 
   enter:
     cmd: ssh -i {{.KEY}} -p 22 root@{{.IP}}

playbooks/destroy.yml

@@ -4,7 +4,7 @@
   vars_files:
     - ../vault.yml
   tasks:
-    - name: Destroy
+    - name: Destroy Terraform.
       community.general.terraform:
         project_path: '../terraform'
         state: "absent"

playbooks/install_k8s.yml

@@ -0,0 +1,237 @@
+# - name: Configure compute for the cluster.
+#   hosts: servers
+#   gather_facts: false
+#   vars:
+#     kubernetes_version: v1.30
+#   tasks:
+#     - name: Download Kubernetes key.
+#       ansible.builtin.apt_key:
+#         url: https://pkgs.k8s.io/core:/stable:/{{ kubernetes_version }}/deb/Release.key
+#         state: present
+
+#     - name: Download Kubernetes repository.
+#       ansible.builtin.apt_repository:
+#         repo: "deb https://pkgs.k8s.io/core:/stable:/{{ kubernetes_version }}/deb/ /"
+#         state: present
+
+#     - name: Download CRI-O key.
+#       ansible.builtin.apt_key:
+#         url: https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/Release.key
+#         state: present
+
+#     - name: Download CRI-O repository.
+#       ansible.builtin.apt_repository:
+#         repo: "deb https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/ /"
+#         state: present
+
+#     - name: Download Helm key.
+#       ansible.builtin.apt_key:
+#         url: https://packages.buildkite.com/helm-linux/helm-debian/gpgkey
+#         state: present
+
+#     - name: Download Helm repository.
+#       ansible.builtin.apt_repository:
+#         repo: "deb https://packages.buildkite.com/helm-linux/helm-debian/any/ any main"
+#         state: present
+
+#     - name: Install packages.
+#       ansible.builtin.apt:
+#         state: present
+#         update_cache: true
+#         name: [cri-o, kubelet, kubeadm, kubectl, python3-pip, helm, git]
+
+#     - name: Install Kubernetes Python packages.
+#       ansible.builtin.pip:
+#         name: [kubernetes, pyyaml]
+#         state: present
+#         break_system_packages: true
+
+#     - name: Enable `br_netfilter` module.
+#       community.general.modprobe:
+#         name: br_netfilter
+#         state: present
+#       notify: Reboot the nodes.
+
+#     - name: Configure `sysctl` permanently.
+#       ansible.posix.sysctl:
+#         name: '{{ item }}'
+#         value: '1'
+#         state: present
+#         reload: true
+#       loop:
+#         - net.bridge.bridge-nf-call-iptables
+#         - net.ipv4.ip_forward
+
+#   handlers:
+#     - name: Reboot the nodes.
+#       ansible.builtin.reboot:
+
+# - name: Spawn new cluster on control node.
+#   hosts: control
+#   gather_facts: false
+#   vars:
+#     config_template: ../templates/InitConfiguration.yml.jinja2
+#     config:
+#       bootstrap_token: "{{ secrets.bootstrap_token }}"
+#       node_ip: 10.0.2.11
+#       node_name: control
+#   vars_files:
+#     - ../vault.yml
+#   tasks:
+#     - name: Test for cluster.
+#       kubernetes.core.k8s_cluster_info:
+#       register: api_status
+#       ignore_errors: true
+
+#     - name: Copy configuration over.
+#       ansible.builtin.template:
+#         src: "{{ config_template }}"
+#         dest: InitConfiguration.yml
+#         mode: preserve
+#       when: "api_status.failed"
+
+#     - name: Initialize cluster.
+#       ansible.builtin.command:
+#         kubeadm init --config InitConfiguration.yml
+#       changed_when: true
+#       when: "api_status.failed"
+
+#     - name: Apply the Kubernetes config to the shell.
+#       ansible.builtin.lineinfile:
+#         path: /etc/environment
+#         line: 'KUBECONFIG=/etc/kubernetes/admin.conf'
+#       when: "api_status.failed"
+
+# - name: Join worker nodes to cluster.
+#   hosts: [node-a, node-b]
+#   vars:
+#     join_template: ../templates/JoinConfiguration.yml.jinja2
+#     join_control_ip: 10.0.2.11
+#     join_bootstrap_token: "{{ secrets.bootstrap_token }}"
+#   vars_files:
+#     - ../vault.yml
+#   tasks:
+#     - name: Copy join configuration over.
+#       vars:
+#         join_worker_ip: "{{ ansible_default_ipv4.address }}"
+#         join_worker_name: "{{ ansible_hostname }}"
+#       ansible.builtin.template:
+#         src: "{{ join_template }}"
+#         dest: JoinConfiguration.yml
+#         mode: preserve
+
+#     - name: Join the nodes.
+#       ansible.builtin.command:
+#         kubeadm join --config JoinConfiguration.yml
+#       changed_when: true
+
+# - name: Install Helm Diff.
+#   gather_facts: false
+#   hosts: control
+#   tasks:
+#     - name: Install it.
+#       kubernetes.core.helm_plugin:
+#         plugin_path: https://github.com/databus23/helm-diff
+#         state: present
+
+- name: Install CNI.
+  gather_facts: false
+  hosts: control
+  tasks:
+    - name: Assign nodes as workers.
+      kubernetes.core.k8s:
+        state: patched
+        kind: Node
+        name: "{{ item }}"
+        definition:
+          metadata:
+            labels:
+              node-role.kubernetes.io/worker: worker
+      loop: [node-a, node-b]
+
+    - name: Create Flannel namespace.
+      kubernetes.core.k8s:
+        state: present
+        kind: Namespace
+        name: kube-flannel
+
+    - name: Add privilege to the namespace.
+      kubernetes.core.k8s:
+        state: patched
+        kind: Namespace
+        name: kube-flannel
+        definition:
+          metadata:
+            labels:
+              pod-security.kubernetes.io/enforce: privileged
+
+    - name: Add Flannel repository.
+      kubernetes.core.helm_repository:
+        name: flannel
+        url: https://flannel-io.github.io/flannel/
+        state: present
+
+    - name: Install Flannel.
+      kubernetes.core.helm:
+        name: flannel
+        chart_ref: flannel/flannel
+        namespace: kube-flannel
+        values:
+          podCidr: 10.244.0.0/16
+        state: present
+
+    - name: Patch CoreDNS deployment.
+      kubernetes.core.k8s_json_patch:
+        name: coredns
+        namespace: kube-system
+        kind: Deployment
+        patch:
+          - op: add
+            path: /spec/template/spec/tolerations/-
+            value:
+              key: node.cloudprovider.kubernetes.io/uninitialized
+              value: "true"
+              effect: NoSchedule
+
+- name: Install `nginx` Controller.
+  gather_facts: false
+  hosts: control
+  vars_files:
+    - ../vault.yml
+    - ../secrets/tf_outputs.yml
+  tasks:
+    - name: Remove schedule taint to nodes.
+      kubernetes.core.k8s_taint:
+        state: absent
+        name: "{{ item }}"
+        taints:
+          - key: node.cloudprovider.kubernetes.io/uninitialized
+            value: true
+            effect: NoSchedule
+      loop: [node-a, node-b]
+
+    - name: Add `ingress-nginx` repository.
+      kubernetes.core.helm_repository:
+        name: ingress-nginx
+        url: https://kubernetes.github.io/ingress-nginx
+        state: present
+
+    - name: Copy over values file.
+      vars:
+        values_template: ../templates/IngressValues.yml.jinja2
+        load_balancer_name: "{{ variables.load_balancer_name }}"
+        network_zone: "{{ variables.network_zone }}"
+        certificate_name: "test"
+      ansible.builtin.template:
+        src: "{{ values_template }}"
+        dest: IngressValues.yml
+        mode: preserve
+
+    - name: Install it.
+      kubernetes.core.helm:
+        name: ingress-nginx-controller
+        chart_ref: ingress-nginx/ingress-nginx
+        namespace: kube-system
+        state: present
+        update_repo_cache: true
+        values_files: [IngressValues.yml]

templates/HCCMValues.yml.jinja2

@@ -0,0 +1,3 @@
+networking:
+  enabled: "true"
+  clusterCIDR: "10.244.0.0/16"

templates/IngressValues.yml.jinja2

@@ -0,0 +1,17 @@
+controller:
+  dnsPolicy: ClusterFirstWithHostNet
+  hostNetwork: true
+  kind: DaemonSet
+  service:
+    annotations:
+      load-balancer.hetzner.cloud/name: {{ load_balancer_name }}
+      load-balancer.hetzner.cloud/type: "lb11"
+      load-balancer.hetzner.cloud/ipv6-disabled: "true"
+      load-balancer.hetzner.cloud/use-private-ip: "true"
+      load-balancer.hetzner.cloud/protocol: "https"
+      load-balancer.hetzner.cloud/network-zone: {{ network_zone }}
+      load-balancer.hetzner.cloud/http-certificates: {{ certificate_name }}
+      load-balancer.hetzner.cloud/http-redirect-http: "true"
+    enableHttp: false
+    targetPorts:
+      https: http

templates/InitConfiguration.yml.jinja2

@@ -0,0 +1,42 @@
+apiVersion: kubeadm.k8s.io/v1beta3
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    token: {{ config.bootstrap_token }}
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: {{ config.node_ip }}
+  bindPort: 6443
+nodeRegistration:
+  criSocket: unix:///var/run/crio/crio.sock
+  imagePullPolicy: IfNotPresent
+  kubeletExtraArgs:
+    cloud-provider: external
+    node-ip: {{ config.node_ip }}
+  name: {{ config.node_name }}
+  taints: null
+---
+apiServer:
+  timeoutForControlPlane: 4m0s
+  certSANs:
+    - {{ config.node_ip }}
+apiVersion: kubeadm.k8s.io/v1beta3
+certificatesDir: /etc/kubernetes/pki
+clusterName: kubernetes
+controllerManager: {}
+dns: {}
+etcd:
+  local:
+    dataDir: /var/lib/etcd
+imageRepository: registry.k8s.io
+kind: ClusterConfiguration
+kubernetesVersion: 1.30.0
+networking:
+  dnsDomain: cluster.local
+  serviceSubnet: 10.96.0.0/12
+  podSubnet: 10.244.0.0/16
+scheduler: {}

templates/JoinConfiguration.yml.jinja2

@@ -0,0 +1,18 @@
+apiVersion: kubeadm.k8s.io/v1beta3
+caCertPath: /etc/kubernetes/pki/ca.crt
+discovery:
+  bootstrapToken:
+    apiServerEndpoint: {{ join_control_ip }}:6443
+    token: {{ join_bootstrap_token }}
+    unsafeSkipCAVerification: true
+  timeout: 5m0s
+  tlsBootstrapToken: {{ join_bootstrap_token }}
+kind: JoinConfiguration
+nodeRegistration:
+  criSocket: unix:///var/run/crio/crio.sock
+  imagePullPolicy: IfNotPresent
+  kubeletExtraArgs:
+    cloud-provider: external
+    node-ip: {{ join_worker_ip }}
+  name: {{ join_worker_name }}
+  taints: null

terraform/.terraform.lock.hcl

@@ -1,28 +1,6 @@
 # This file is maintained automatically by "terraform init".
 # Manual edits may be lost in future updates.
 
-provider "registry.terraform.io/hashicorp/aws" {
-  version = "6.27.0"
-  hashes = [
-    "h1:emgTfB1LXSFYh9uAwgsRMoMIN5Wz7jNNKq3rqC0EHWk=",
-    "zh:177a24b806c72e8484b5cabc93b2b38e3d770ae6f745a998b54d6619fd0e8129",
-    "zh:4ac4a85c14fb868a3306b542e6a56c10bd6c6d5a67bc0c9b8f6a9060cf5f3be7",
-    "zh:552652185bc85c8ba1da1d65dea47c454728a5c6839c458b6dcd3ce71c19ccfc",
-    "zh:60284b8172d09aee91eae0856f09855eaf040ce3a58d6933602ae17c53f8ed04",
-    "zh:6be38d156756ca61fb8e7c752cc5d769cd709686700ac4b230f40a6e95b5dbc9",
-    "zh:7a409138fae4ef42e3a637e37cb9efedf96459e28a3c764fc4e855e8db9a7485",
-    "zh:8070cf5224ed1ed3a3e9a59f7c30ff88bf071c7567165275d477c1738a56c064",
-    "zh:894439ef340a9a79f69cd759e27ad11c7826adeca27be1b1ca82b3c9702fa300",
-    "zh:89d035eebf08a97c89374ff06040955ddc09f275ecca609d0c9d58d149bef5cf",
-    "zh:985b1145d724fc1f38369099e4a5087141885740fd6c0b1dbc492171e73c2e49",
-    "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
-    "zh:a80b47ae8d1475201c86bd94a5dcb9dd4da5e8b73102a90820b68b66b76d50fd",
-    "zh:d3395be1556210f82199b9166a6b2e677cee9c4b67e96e63f6c3a98325ad7ab0",
-    "zh:db0b869d09657f6f1e4110b56093c5fcdf9dbdd97c020db1e577b239c0adcbce",
-    "zh:ffc72e680370ae7c21f9bd3082c6317730df805c6797427839a6b6b7e9a26a01",
-  ]
-}
-
 provider "registry.terraform.io/hetznercloud/hcloud" {
   version     = "1.57.0"
   constraints = "~> 1.45"

terraform/compute.tf

@@ -51,3 +51,18 @@ resource "hcloud_server" "server" {
 
   depends_on = [hcloud_network_subnet.subnet]
 }
+
+resource "hcloud_load_balancer" "lb" {
+  name               = "lb-hetzner"
+  load_balancer_type = "lb11"
+  network_zone       = "eu-central"
+}
+
+resource "hcloud_load_balancer_target" "load_balancer_target" {
+  for_each = hcloud_server.server
+
+  type             = "server"
+  load_balancer_id = hcloud_load_balancer.lb.id
+  use_private_ip   = true
+  server_id        = each.value.id
+}

terraform/network.tf

@@ -12,13 +12,6 @@ resource "hcloud_network_subnet" "subnet" {
   ip_range     = local.subnet-cidr
 }
 
-// Attach the load balancer to the compute subnet.
-resource "hcloud_load_balancer_network" "attachment" {
-  load_balancer_id = hcloud_load_balancer.lb.id
-  subnet_id        = hcloud_network_subnet.subnet.id
-  ip               = local.lb-private-ip
-}
-
 // Provide internet to the private servers, by sending all internet traffic to
 // the NAT.
 resource "hcloud_network_route" "gateway" {
@@ -27,8 +20,9 @@ resource "hcloud_network_route" "gateway" {
   gateway     = local.nat-private-ip
 }
 
-// A managed certificate for the domain, to be used by the load balancer.
-resource "hcloud_managed_certificate" "managed_cert" {
-  name         = "managed_cert"
-  domain_names = ["*.${local.domain}", "${local.domain}"]
+// Attach the load blaancer to the private network.
+resource "hcloud_load_balancer_network" "attachment" {
+  load_balancer_id = hcloud_load_balancer.lb.id
+  subnet_id        = hcloud_network_subnet.subnet.id
+  ip               = local.lb-private-ip
 }

terraform/outputs.tf

@@ -1,3 +1,7 @@
 output "nat_public_ip" {
   value = hcloud_server.nat.ipv4_address
 }
+
+output "private_network_id" {
+  value = hcloud_network.net.id
+}

terraform/routing.tf

@@ -0,0 +1,32 @@
+data "hcloud_zone" "zone" {
+  name = local.domain
+}
+
+// Attach the load balancer to the domain.
+resource "hcloud_zone_rrset" "records" {
+  for_each = toset(["@", "*"])
+  zone     = data.hcloud_zone.zone.name
+  name     = each.value
+  type     = "A"
+  ttl      = 60
+  records  = [{ value = hcloud_load_balancer.lb.ipv4 }]
+
+  change_protection = false
+}
+
+// A managed certificate for the domain, to be used by the load balancer.
+resource "hcloud_managed_certificate" "main" {
+  name         = local.certificate_name
+  domain_names = ["*.${local.domain}", "${local.domain}"]
+}
+
+resource "hcloud_load_balancer_service" "load_balancer_service" {
+  load_balancer_id = hcloud_load_balancer.lb.id
+  protocol         = "https"
+
+  http {
+    sticky_sessions = true
+    certificates    = [hcloud_managed_certificate.main.id]
+    redirect_http   = true
+  }
+}

terraform/variables.tf

@@ -11,6 +11,8 @@ locals {
   }
 
   domain = "maximhutz.com"
+
+  certificate_name = "Main Certificate"
 }
 
 variable "public_key_file" {

vault.yml

@@ -1,36 +1,40 @@
 $ANSIBLE_VAULT;1.1;AES256
-35323436626135616534616432353038616532613138393165366534663361636633303336643766
-6538656163363065363832333961653736373231366230340a363539376639323635303131343966
-38626563666630663934333633303636323964666339353035396566326634336134666436383061
-3838623462363565640a383832306334646530326133653235323730386563343462316332333236
-65646265356132343761323661636235363838643838343936353862646336376461633433666364
-36613763346439636263303166323662633237353533363064613437636561616538343832656531
-31323064663534363234303633386562313565366366366533383761663466326333393135343534
-31646263333736353132356136343765396362373864616261363666636262313631303663653765
-64356133656536333366663237366562343266383462323630346434363065303466373035386131
-63356434396433353731656233346465656536303437656232353831373062386161313165326164
-61393031613733396131643762363465303833326633326364343631383733383966343965306530
-65316638633132633562316637373166623631636237346435613733336333616138323534633266
-33623362653166646639373137396265353064326435646138336635383232663437336238343366
-66613735306235353563383762383763653964313766616539336362616332663138353233326261
-36313631663832633163633966613864323331356362376437393563323061303864646665623332
-65363236323339346232303763326636333935306363323164626635663833356661386532303430
-31346133313639356361316563343761623466633030633030333138353538656633643630653862
-34633638666562656434346636613136323461353565323133613461646638366135353638393834
-36373030643030303365353531313665643666383433333562626532323363366432336333623733
-62306462616538313139646437333363616364343766636562366235613635373335613164336361
-33313130353361373164643833636231383233323034313861343134363964653031396530643537
-36626561303839653234313466356330633666386166623335623864653935316566366539626238
-62663438643730653463343634626361633766343366626266386538326136323932626636313864
-66393465646366633036396538336662643230646631623436386462383434643665383233393930
-38306265393832386262616261643863306438656662666333306662366464393736623939656565
-33353637353861636562323639306562343235316566353130663563623538656131363132356231
-61636239346138353833633065633030653138306262633830303739356662353063363032656439
-34623336336233626362316638663335613639363533393833623533306334636532623264323966
-64393234656463643438303630306565353534663066363065383533336130323434393131326166
-61636538356265343937626435653763366333326364376461616138313637316535636238376263
-63316430323863316366653338653261373438626563643433346133363031313464323964326230
-30626461333437316332616462373334643736623930666432343966623436623834613038626137
-35613730303166303739316132306164383965636532663563613166373337363030636630346164
-63396166353534373338626261366437653662353166366263653562316562313433616232323636
-373163323937626435303963323632343631
+62656134326239313562396138346634316530303635353562616163323836666132616565366336
+3035353733653832316165356663303264396439393232390a666465306363356335383734616438
+37313833663535356163616666343933303363383462353064633538333433373431663161626230
+3962303165346162360a626536313165643466343965633431343833653163656266396535656232
+35653139613737336431323733616533363531616131613965663534343938396661336331376633
+62306130323131626435303262326261376630616433613363363536663638306261643734363661
+66366631393034653536343163313862623733316465376533313030393761363033376536643861
+63313735343033656332333838343532343236623435303135383033306131313930316137613634
+37386339313530353534343162613733333935333136656134623862323861653739353636366363
+38656565643437663330353366636331316337626438323162393838346534393063386338326336
+64373030336466376432386334653737313461626264396431613330393938316230623235663962
+62323431626261386238363163646662336134373534376632653431396532626438613830396164
+34663434656131336265353336633930666230323131633130373833396230313634646134353464
+30373537623939316565393966376439336465623330353037303536306632646361643437306139
+36393232623236613737336263396138376336396335316465663661613635636232383435666230
+65333361656337653135363239346264613530626231636635303466326331323832383337626534
+64306630306531393461356535336136323833643735353232343336623830656563616663353933
+34656562626238343030383833326333323463306634616333303531633832326532316664383837
+65343463323837376630323663323961636631376535313538646462626130653431306563323137
+35616335333265306366376532353861643935663764326334313035323432343361306639643633
+62643932303161326634656463633166643062363262303665633261303730353438633834326432
+63386439653266333561336432653737316538333330613662356535363162633635663039646430
+31363866396265613639333266636532373438366430663632633061663736366366623061313765
+37313932613339643731616263656636303439633637623935333136353866303361396230393632
+37316566303932336361653335353632353161353864616361326665393065363736363430666464
+62656333393632313664653837393335353662363965313238633131313631373534313336613831
+33313762316330653835616637323134656536626661343833373336363430633836663831643563
+39303364656638306661616537623538663230326639643533306538353435626336383435633836
+35656633313436623733666464346337343664393236336535616135333032363034666333316233
+65363537633630356662353034613935366330643361393631353561643062376239343363646462
+38633335356234396334313265393235636337663365646533343234323634646166623038343266
+32646432653731383366616333633862643531303633613136386331383365376633343935666563
+33363035356365626263646132353631653336383939646538393336393463626632663661663962
+63656238353463356665633964316135646264333262633862643234313035386230666661643733
+65396534636365356130356463393634646136373362343334636138633531383135333637323635
+35366131353261643661373366653838373238343732633430653862613134386565303765326166
+32386465336231666564653361653235646231623065643738613939353439323430656236613633
+63333034303863633036613662313238383430373365353637323062363363303461333766373164
+393133613238363662663335626561393630