ceph experiments

roles/ceph/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+- name: import keyring to admin
+  command:
+    cmd: ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring
+- name: import keyring to osd
+  command:
+    cmd: ceph-authtool /etc/ceph/ceph.mon.keyring --import-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring

roles/ceph/tasks/main.yml

@@ -0,0 +1,70 @@
+---
+- name: install ceph utilities
+  apt:
+    name: "{{item}}"
+  loop:
+    - ceph
+    - ceph-mds
+    - lvm2
+- name: setup ceph monitor
+  block:
+   - name: configure ceph monitor host
+     template:
+       src: ceph.conf.j2
+       dest: /etc/ceph/ceph.conf
+   - name: create ceph mon keyring
+     command:
+       cmd: ceph-authtool --create-keyring /etc/ceph/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'
+     args:
+       creates: /etc/ceph/ceph.mon.keyring
+     notify: import keyring to admin
+   - name: create ceph admin keyring
+     command:
+       cmd: ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+     args:
+       creates: /etc/ceph/ceph.client.admin.keyring
+   - name: create ceph bootstrap-osd keyring
+     command:
+       cmd: ceph-authtool --create-keyring /var/lib/ceph/bootstrap-osd/ceph.keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
+     args:
+       creates: /var/lib/ceph/bootstrap-osd/ceph.keyring
+     notify: import keyring to osd
+   - name: flush handlers
+     meta: flush_handlers
+   - name: change ownership of mon keyring
+     file:
+       path: /etc/ceph/ceph.mon.keyring
+       owner: ceph
+       group: ceph
+   - name: generate monitor map
+     shell:
+       cmd: monmaptool --create --add {{ansible_nodename}} {{wg_ip}} --fsid {{'zkntceph'|to_uuid}} /etc/ceph/monmap
+     args:
+       creates: /etc/ceph/monmap
+   - name: create data directory
+     file:
+       path: /var/lib/ceph/mon/ceph-{{ansible_nodename}}
+       state: directory
+       owner: ceph
+       group: ceph
+   - name: populate monitor daemon
+     command:
+       cmd: ceph-mon --mkfs -i {{ansible_nodename}} --monmap /etc/ceph/monmap --keyring /etc/ceph/ceph.mon.keyring
+     args:
+       creates: /var/lib/ceph/mon/ceph-{{ansible_nodename}}kv_backend
+     become_user: ceph
+   - name: start and enable ceph-mon
+     systemd:
+       name: ceph-mon@node01
+       enabled: yes
+       state: started
+  when: ansible_hostname == "node01"
+- name: set up ceph manager
+  block:
+    - name: create authentication key
+      shell:
+        cmd: ceph auth get-or-create mgr.{{ansible_nodename}} mon 'allow profile mgr' osd 'allow *' mds 'allow *' > /var/lib/ceph/mgr/ceph-{{ansible_nodename}}
+      args:
+        creates: /var/lib/ceph/mgr/ceph-{{ansible_nodename}}
+  when: ansible_hostname == "node01"
+

roles/ceph/templates/ceph.conf.j2

@@ -0,0 +1,14 @@
+[global]
+fsid = {{ 'zkntceph' | to_uuid }}
+mon initial members = node01
+mon host = 10.23.23.11
+public network = 10.23.23.0/24
+auth cluster required = cephx
+auth service required = cephx
+auth client required = cephx
+osd journal size = 1024
+osd pool default size = 3
+osd pool default min size = 2
+osd pool default pg num = 333
+osd pool default pgp num = 333
+osd crush chooseleaf type = 1

roles/etcd/tasks/main.yaml

@@ -5,5 +5,5 @@
 - name: etcd configuration
   template:
     src: defaults-etcd.j2
-    dest: /etc/defaults/etcd
+    dest: /etc/default/etcd
   notify: restart etcd

roles/gluster/handlers/main.yaml

@@ -0,0 +1,5 @@
+---
+- name: restart rpcbind
+  service:
+    name: rpcbind
+    state: restarted

roles/gluster/tasks/main.yaml

@@ -1,4 +1,40 @@
 ---
+- name: install rpcbind
+  apt:
+    name: rpcbind
+- name: disable rpcbind from the internet
+  iptables:
+    action: insert
+    chain: INPUT
+    comment: drop portmapper
+    in_interface: "!nodevpn"
+    jump: DROP
+    protocol: "{{item}}"
+    destination_port: "111"
+  with_items:
+    - tcp
+    - udp
+- name: bind rpcbind to vpn ip
+  lineinfile:
+     create: yes
+     path: /etc/rpcbind.conf
+     line: "OPTIONS=-w -h {{wg_ip}}"
+     regexp: "^OPTIONS.*"
+  notify: restart rpcbind
+- name: add vpn to hosts.allow
+  lineinfile:
+     path: /etc/hosts.allow
+     line: "rpcbind: 10.0.0.0/8"
+     regexp: "rpbcind.*"
+  notify: restart rpcbind
+- name: block everyone
+  lineinfile:
+     path: /etc/hosts.deny
+     line: "rpcbind: ALL"
+     regexp: "rpcbind.*"
+  notify: restart rpcbind
+- name: fire handlers
+  meta: flush_handlers
 - name: install gluster package
   apt:
     name: glusterfs-server
@@ -15,25 +51,29 @@
   mount:
     path: /mnt/gluster
     src: /dev/sdb
-    options: noatime
+    opts: noatime
     fstype: ext4
     state: mounted
-- name: gluster peers
-  gluster_peer:
-    nodes:
-      - node01
-      - node02
-      - node03
-- name: gluster volume
-  gluster_volume:
-    name: data
-    replicas: 3
-    state: started
-    bricks: /mnt/gluster/data
-    cluster:
-      - node01
-      - node02
-      - node03
+- name: bootstrap gluster cluster
+  block:
+  - name: gluster peers
+    gluster_peer:
+      nodes:
+        - node01
+        - node02
+        - node03
+  - name: gluster volume
+    gluster_volume:
+      name: data
+      replicas: 3
+      state: present
+      bricks: /mnt/gluster/data
+      start_on_create: yes
+      cluster:
+        - node01
+        - node02
+        - node03
+  when: ansible_nodename == "node01"
 - name: gluster client package
   apt:
      name: glusterfs-client
@@ -41,6 +81,6 @@
   mount:
     path: /data
     src: "{{ansible_nodename}}:/data"
-    options: noatime
+    opts: noatime
     fstype: glusterfs
     state: mounted

roles/k3s/tasks/main.yaml

@@ -10,3 +10,8 @@
     dest: /etc/systemd/system/k3s.service
   notify: start k3s
 - name: enable and start k3s
+  systemd:
+    daemon_reload: yes
+    enabled: yes
+    name: k3s
+    state: restarted

roles/k3s/templates/service-k3s.j2

@@ -5,4 +5,4 @@ After=network-online.target
 [Service]
 Environment=K3S_TOKEN={{k3s_token}}
 Environment=K3S_DATASTORE_ENDPOINT=http://node01:2379,http://node02:2379,http://node03:2379
-Exec=/usr/local/sbin/k3s server --no-deploy traefik --bind-address {{wg_ip}} --advertise_address {{wg_ip}}
+ExecStart=/usr/local/sbin/k3s server --no-deploy traefik --bind-address {{wg_ip}} --advertise-address {{wg_ip}}

roles/wireguard/tasks/main.yaml

@@ -35,3 +35,5 @@
     path: /etc/hosts
     line: "{{item.value.wg_ip}} {{item.key}}"
   loop: "{{hostvars|dict2items}}"
+- name: fire handlers
+  meta: flush_handlers

site.yml

@@ -4,6 +4,7 @@
   roles:
     - base
     - wireguard
-    - gluster
-    - etcd
-    - k3s
+    - ceph
+    #- gluster
+    #- etcd
+    #- k3s