From e7af4bf4a35cfb2bd07b7484aaf4e2a7b572c634 Mon Sep 17 00:00:00 2001 From: David Landry Date: Sun, 21 Sep 2025 15:25:00 -0400 Subject: [PATCH] chore(k8s-mgmt): splitting some resource in prep for dual clusters --- bootstrap/cluster-resources/in-cluster.json | 1 - .../rke2-system-upgrade-controller.yaml | 264 ------- .../in-cluster/rke2-system-upgrade-crd.yaml | 642 ------------------ .../in-cluster/rke2-upgrade-plan.yaml | 64 -- bootstrap/cluster-resources/k8s-home.json | 4 + .../{in-cluster => k8s-home}/cnpg-1.24.0.yaml | 0 bootstrap/cluster-resources/k8s-mgmt.json | 4 + .../{in-cluster => k8s-mgmt}/README.md | 0 .../{in-cluster => k8s-mgmt}/argocd-ns.yaml | 0 .../machineregistration.elemental.yaml | 0 10 files changed, 8 insertions(+), 971 deletions(-) delete mode 100644 bootstrap/cluster-resources/in-cluster.json delete mode 100644 bootstrap/cluster-resources/in-cluster/rke2-system-upgrade-controller.yaml delete mode 100644 bootstrap/cluster-resources/in-cluster/rke2-system-upgrade-crd.yaml delete mode 100644 bootstrap/cluster-resources/in-cluster/rke2-upgrade-plan.yaml create mode 100644 bootstrap/cluster-resources/k8s-home.json rename bootstrap/cluster-resources/{in-cluster => k8s-home}/cnpg-1.24.0.yaml (100%) create mode 100644 bootstrap/cluster-resources/k8s-mgmt.json rename bootstrap/cluster-resources/{in-cluster => k8s-mgmt}/README.md (100%) rename bootstrap/cluster-resources/{in-cluster => k8s-mgmt}/argocd-ns.yaml (100%) rename bootstrap/cluster-resources/{in-cluster => k8s-mgmt}/machineregistration.elemental.yaml (100%) diff --git a/bootstrap/cluster-resources/in-cluster.json b/bootstrap/cluster-resources/in-cluster.json deleted file mode 100644 index 259083a..0000000 --- a/bootstrap/cluster-resources/in-cluster.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"in-cluster","server":"https://kubernetes.default.svc"} \ No newline at end of file diff --git a/bootstrap/cluster-resources/in-cluster/rke2-system-upgrade-controller.yaml b/bootstrap/cluster-resources/in-cluster/rke2-system-upgrade-controller.yaml deleted file mode 100644 index 0b0f9ee..0000000 --- a/bootstrap/cluster-resources/in-cluster/rke2-system-upgrade-controller.yaml +++ /dev/null @@ -1,264 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system-upgrade-controller -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - namespaces - - nodes - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - update -- apiGroups: - - upgrade.cattle.io - resources: - - plans - - plans/status - verbs: - - get - - list - - watch - - create - - patch - - update - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: system-upgrade-controller - namespace: system-upgrade -rules: -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - deletecollection - - patch - - update - - get - - list - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: system-upgrade-controller-drainer -rules: -- apiGroups: - - "" - resources: - - pods/eviction - verbs: - - create -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - patch -- apiGroups: - - apps - resources: - - statefulsets - - daemonsets - - replicasets - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system-upgrade-drainer -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system-upgrade-controller-drainer -subjects: -- kind: ServiceAccount - name: system-upgrade - namespace: system-upgrade ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: system-upgrade -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system-upgrade-controller -subjects: -- kind: ServiceAccount - name: system-upgrade - namespace: system-upgrade ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: system-upgrade - namespace: system-upgrade -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: system-upgrade-controller -subjects: -- kind: ServiceAccount - name: system-upgrade - namespace: system-upgrade ---- -apiVersion: v1 -kind: Namespace -metadata: - labels: - pod-security.kubernetes.io/enforce: privileged - name: system-upgrade ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: system-upgrade - namespace: system-upgrade ---- -apiVersion: v1 -data: - SYSTEM_UPGRADE_CONTROLLER_DEBUG: "false" - SYSTEM_UPGRADE_CONTROLLER_THREADS: "2" - SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: "900" - SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: "99" - SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: Always - SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: rancher/kubectl:v1.25.4 - SYSTEM_UPGRADE_JOB_PRIVILEGED: "true" - SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: "900" - SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m -kind: ConfigMap -metadata: - name: default-controller-env - namespace: system-upgrade ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: system-upgrade-controller - namespace: system-upgrade -spec: - selector: - matchLabels: - upgrade.cattle.io/controller: system-upgrade-controller - template: - metadata: - labels: - upgrade.cattle.io/controller: system-upgrade-controller - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - containers: - - env: - - name: SYSTEM_UPGRADE_CONTROLLER_NAME - valueFrom: - fieldRef: - fieldPath: metadata.labels['upgrade.cattle.io/controller'] - - name: SYSTEM_UPGRADE_CONTROLLER_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - envFrom: - - configMapRef: - name: default-controller-env - image: rancher/system-upgrade-controller:v0.13.4 - imagePullPolicy: IfNotPresent - name: system-upgrade-controller - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsGroup: 65534 - runAsNonRoot: true - runAsUser: 65534 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /etc/ssl - name: etc-ssl - readOnly: true - - mountPath: /etc/pki - name: etc-pki - readOnly: true - - mountPath: /etc/ca-certificates - name: etc-ca-certificates - readOnly: true - - mountPath: /tmp - name: tmp - serviceAccountName: system-upgrade - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/controlplane - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoExecute - key: node-role.kubernetes.io/etcd - operator: Exists - volumes: - - hostPath: - path: /etc/ssl - type: DirectoryOrCreate - name: etc-ssl - - hostPath: - path: /etc/pki - type: DirectoryOrCreate - name: etc-pki - - hostPath: - path: /etc/ca-certificates - type: DirectoryOrCreate - name: etc-ca-certificates - - emptyDir: {} - name: tmp diff --git a/bootstrap/cluster-resources/in-cluster/rke2-system-upgrade-crd.yaml b/bootstrap/cluster-resources/in-cluster/rke2-system-upgrade-crd.yaml deleted file mode 100644 index 6fb5343..0000000 --- a/bootstrap/cluster-resources/in-cluster/rke2-system-upgrade-crd.yaml +++ /dev/null @@ -1,642 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: plans.upgrade.cattle.io -spec: - group: upgrade.cattle.io - names: - categories: - - upgrade - kind: Plan - plural: plans - singular: plan - preserveUnknownFields: false - scope: Namespaced - versions: - - additionalPrinterColumns: - - jsonPath: .spec.upgrade.image - name: Image - type: string - - jsonPath: .spec.channel - name: Channel - type: string - - jsonPath: .spec.version - name: Version - type: string - name: v1 - schema: - openAPIV3Schema: - properties: - spec: - properties: - channel: - nullable: true - type: string - concurrency: - type: integer - cordon: - type: boolean - drain: - nullable: true - properties: - deleteEmptydirData: - nullable: true - type: boolean - deleteLocalData: - nullable: true - type: boolean - disableEviction: - type: boolean - force: - type: boolean - gracePeriod: - nullable: true - type: integer - ignoreDaemonSets: - nullable: true - type: boolean - podSelector: - nullable: true - properties: - matchExpressions: - items: - properties: - key: - nullable: true - type: string - operator: - nullable: true - type: string - values: - items: - nullable: true - type: string - nullable: true - type: array - type: object - nullable: true - type: array - matchLabels: - additionalProperties: - nullable: true - type: string - nullable: true - type: object - type: object - skipWaitForDeleteTimeout: - type: integer - timeout: - nullable: true - type: integer - type: object - exclusive: - type: boolean - imagePullSecrets: - items: - properties: - name: - nullable: true - type: string - type: object - nullable: true - type: array - jobActiveDeadlineSecs: - type: integer - nodeSelector: - nullable: true - properties: - matchExpressions: - items: - properties: - key: - nullable: true - type: string - operator: - nullable: true - type: string - values: - items: - nullable: true - type: string - nullable: true - type: array - type: object - nullable: true - type: array - matchLabels: - additionalProperties: - nullable: true - type: string - nullable: true - type: object - type: object - prepare: - nullable: true - properties: - args: - items: - nullable: true - type: string - nullable: true - type: array - command: - items: - nullable: true - type: string - nullable: true - type: array - envFrom: - items: - properties: - configMapRef: - nullable: true - properties: - name: - nullable: true - type: string - optional: - nullable: true - type: boolean - type: object - prefix: - nullable: true - type: string - secretRef: - nullable: true - properties: - name: - nullable: true - type: string - optional: - nullable: true - type: boolean - type: object - type: object - nullable: true - type: array - envs: - items: - properties: - name: - nullable: true - type: string - value: - nullable: true - type: string - valueFrom: - nullable: true - properties: - configMapKeyRef: - nullable: true - properties: - key: - nullable: true - type: string - name: - nullable: true - type: string - optional: - nullable: true - type: boolean - type: object - fieldRef: - nullable: true - properties: - apiVersion: - nullable: true - type: string - fieldPath: - nullable: true - type: string - type: object - resourceFieldRef: - nullable: true - properties: - containerName: - nullable: true - type: string - divisor: - nullable: true - type: string - resource: - nullable: true - type: string - type: object - secretKeyRef: - nullable: true - properties: - key: - nullable: true - type: string - name: - nullable: true - type: string - optional: - nullable: true - type: boolean - type: object - type: object - type: object - nullable: true - type: array - image: - nullable: true - type: string - securityContext: - nullable: true - properties: - allowPrivilegeEscalation: - nullable: true - type: boolean - capabilities: - nullable: true - properties: - add: - items: - nullable: true - type: string - nullable: true - type: array - drop: - items: - nullable: true - type: string - nullable: true - type: array - type: object - privileged: - nullable: true - type: boolean - procMount: - nullable: true - type: string - readOnlyRootFilesystem: - nullable: true - type: boolean - runAsGroup: - nullable: true - type: integer - runAsNonRoot: - nullable: true - type: boolean - runAsUser: - nullable: true - type: integer - seLinuxOptions: - nullable: true - properties: - level: - nullable: true - type: string - role: - nullable: true - type: string - type: - nullable: true - type: string - user: - nullable: true - type: string - type: object - seccompProfile: - nullable: true - properties: - localhostProfile: - nullable: true - type: string - type: - nullable: true - type: string - type: object - windowsOptions: - nullable: true - properties: - gmsaCredentialSpec: - nullable: true - type: string - gmsaCredentialSpecName: - nullable: true - type: string - hostProcess: - nullable: true - type: boolean - runAsUserName: - nullable: true - type: string - type: object - type: object - volumes: - items: - properties: - destination: - nullable: true - type: string - name: - nullable: true - type: string - source: - nullable: true - type: string - type: object - nullable: true - type: array - type: object - secrets: - items: - properties: - ignoreUpdates: - type: boolean - name: - nullable: true - type: string - path: - nullable: true - type: string - type: object - nullable: true - type: array - serviceAccountName: - nullable: true - type: string - tolerations: - items: - properties: - effect: - nullable: true - type: string - key: - nullable: true - type: string - operator: - nullable: true - type: string - tolerationSeconds: - nullable: true - type: integer - value: - nullable: true - type: string - type: object - nullable: true - type: array - upgrade: - nullable: true - properties: - args: - items: - nullable: true - type: string - nullable: true - type: array - command: - items: - nullable: true - type: string - nullable: true - type: array - envFrom: - items: - properties: - configMapRef: - nullable: true - properties: - name: - nullable: true - type: string - optional: - nullable: true - type: boolean - type: object - prefix: - nullable: true - type: string - secretRef: - nullable: true - properties: - name: - nullable: true - type: string - optional: - nullable: true - type: boolean - type: object - type: object - nullable: true - type: array - envs: - items: - properties: - name: - nullable: true - type: string - value: - nullable: true - type: string - valueFrom: - nullable: true - properties: - configMapKeyRef: - nullable: true - properties: - key: - nullable: true - type: string - name: - nullable: true - type: string - optional: - nullable: true - type: boolean - type: object - fieldRef: - nullable: true - properties: - apiVersion: - nullable: true - type: string - fieldPath: - nullable: true - type: string - type: object - resourceFieldRef: - nullable: true - properties: - containerName: - nullable: true - type: string - divisor: - nullable: true - type: string - resource: - nullable: true - type: string - type: object - secretKeyRef: - nullable: true - properties: - key: - nullable: true - type: string - name: - nullable: true - type: string - optional: - nullable: true - type: boolean - type: object - type: object - type: object - nullable: true - type: array - image: - nullable: true - type: string - securityContext: - nullable: true - properties: - allowPrivilegeEscalation: - nullable: true - type: boolean - capabilities: - nullable: true - properties: - add: - items: - nullable: true - type: string - nullable: true - type: array - drop: - items: - nullable: true - type: string - nullable: true - type: array - type: object - privileged: - nullable: true - type: boolean - procMount: - nullable: true - type: string - readOnlyRootFilesystem: - nullable: true - type: boolean - runAsGroup: - nullable: true - type: integer - runAsNonRoot: - nullable: true - type: boolean - runAsUser: - nullable: true - type: integer - seLinuxOptions: - nullable: true - properties: - level: - nullable: true - type: string - role: - nullable: true - type: string - type: - nullable: true - type: string - user: - nullable: true - type: string - type: object - seccompProfile: - nullable: true - properties: - localhostProfile: - nullable: true - type: string - type: - nullable: true - type: string - type: object - windowsOptions: - nullable: true - properties: - gmsaCredentialSpec: - nullable: true - type: string - gmsaCredentialSpecName: - nullable: true - type: string - hostProcess: - nullable: true - type: boolean - runAsUserName: - nullable: true - type: string - type: object - type: object - volumes: - items: - properties: - destination: - nullable: true - type: string - name: - nullable: true - type: string - source: - nullable: true - type: string - type: object - nullable: true - type: array - type: object - version: - nullable: true - type: string - required: - - upgrade - type: object - status: - properties: - applying: - items: - nullable: true - type: string - nullable: true - type: array - conditions: - items: - properties: - lastTransitionTime: - nullable: true - type: string - lastUpdateTime: - nullable: true - type: string - message: - nullable: true - type: string - reason: - nullable: true - type: string - status: - nullable: true - type: string - type: - nullable: true - type: string - type: object - nullable: true - type: array - latestHash: - nullable: true - type: string - latestVersion: - nullable: true - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/bootstrap/cluster-resources/in-cluster/rke2-upgrade-plan.yaml b/bootstrap/cluster-resources/in-cluster/rke2-upgrade-plan.yaml deleted file mode 100644 index 21e33e9..0000000 --- a/bootstrap/cluster-resources/in-cluster/rke2-upgrade-plan.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# Server plan -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: server-plan - namespace: system-upgrade - labels: - rke2-upgrade: server -spec: - concurrency: 1 - nodeSelector: - matchExpressions: - # When using k8s version 1.19 or older, swap control-plane with master - - { - key: node-role.kubernetes.io/control-plane, - operator: In, - values: ["true"], - } - tolerations: - - key: "CriticalAddonsOnly" - operator: "Equal" - value: "true" - effect: "NoExecute" - serviceAccountName: system-upgrade - cordon: true - # drain: - # force: true - upgrade: - image: rancher/rke2-upgrade - version: v1.30.7-rke2r1 ---- -# Agent plan -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: agent-plan - namespace: system-upgrade - labels: - rke2-upgrade: agent -spec: - concurrency: 2 - nodeSelector: - matchExpressions: - - { key: beta.kubernetes.io/os, operator: In, values: ["linux"] } - - { key: rke2-upgrade, operator: Exists } - - { key: rke2-upgrade, operator: NotIn, values: ["disabled", "false"] } - # When using k8s version 1.19 or older, swap control-plane with master - - { - key: node-role.kubernetes.io/control-plane, - operator: NotIn, - values: ["true"], - } - prepare: - args: - - prepare - - server-plan - image: rancher/rke2-upgrade - serviceAccountName: system-upgrade - cordon: true - drain: - force: true - upgrade: - image: rancher/rke2-upgrade - version: v1.30.7-rke2r1 diff --git a/bootstrap/cluster-resources/k8s-home.json b/bootstrap/cluster-resources/k8s-home.json new file mode 100644 index 0000000..db69f02 --- /dev/null +++ b/bootstrap/cluster-resources/k8s-home.json @@ -0,0 +1,4 @@ +{ + "name": "k8s-home", + "server": "https://elli" +} diff --git a/bootstrap/cluster-resources/in-cluster/cnpg-1.24.0.yaml b/bootstrap/cluster-resources/k8s-home/cnpg-1.24.0.yaml similarity index 100% rename from bootstrap/cluster-resources/in-cluster/cnpg-1.24.0.yaml rename to bootstrap/cluster-resources/k8s-home/cnpg-1.24.0.yaml diff --git a/bootstrap/cluster-resources/k8s-mgmt.json b/bootstrap/cluster-resources/k8s-mgmt.json new file mode 100644 index 0000000..a88f329 --- /dev/null +++ b/bootstrap/cluster-resources/k8s-mgmt.json @@ -0,0 +1,4 @@ +{ + "name": "k8s-mgmt", + "server": "https://kubernetes.default.svc" +} diff --git a/bootstrap/cluster-resources/in-cluster/README.md b/bootstrap/cluster-resources/k8s-mgmt/README.md similarity index 100% rename from bootstrap/cluster-resources/in-cluster/README.md rename to bootstrap/cluster-resources/k8s-mgmt/README.md diff --git a/bootstrap/cluster-resources/in-cluster/argocd-ns.yaml b/bootstrap/cluster-resources/k8s-mgmt/argocd-ns.yaml similarity index 100% rename from bootstrap/cluster-resources/in-cluster/argocd-ns.yaml rename to bootstrap/cluster-resources/k8s-mgmt/argocd-ns.yaml diff --git a/bootstrap/cluster-resources/in-cluster/machineregistration.elemental.yaml b/bootstrap/cluster-resources/k8s-mgmt/machineregistration.elemental.yaml similarity index 100% rename from bootstrap/cluster-resources/in-cluster/machineregistration.elemental.yaml rename to bootstrap/cluster-resources/k8s-mgmt/machineregistration.elemental.yaml