feat(warehouses): Add support to backfill manually exported missing electricity data

infra/ansible/group_vars/all/all.yml

@@ -33,7 +33,7 @@ keycloak_realm:
   display_name: "ISIS Analytics Data Platform"
 keycloak_realm_url: "{{ keycloak_url }}/realms/{{ keycloak_realm.name }}"
 keycloak_bootstrap:
-  admin_user: "{{ secrets_keycloak['bootstrap_admin_user'] }}"
+  admin_user: "temp-admin"
   admin_password: "{{ secrets_keycloak['bootstrap_admin_password'] }}"
 
 lakekeeper_base_path: /iceberg

infra/ansible/group_vars/keycloak.yml

@@ -5,6 +5,10 @@ keycloak_db_name: "{{ secrets_keycloak['db_name'] }}"
 keycloak_db_user: "{{ secrets_keycloak['db_user'] }}"
 keycloak_db_password: "{{ secrets_keycloak['db_password'] }}"
 
+keycloak_local_admin:
+  user: "{{ secrets_keycloak['master_local_admin_user'] }}"
+  password: "{{ secrets_keycloak['master_local_admin_password'] }}"
+
 # Eveything is assigned to the realm defined by keycloak_realm.name in all/all.yml
 keycloak_client_scopes:
   - name: lakekeeper

infra/ansible/roles/keycloak/tasks/main.yml

@@ -44,6 +44,24 @@
     path: "{{ keycloak_working_dir }}"
     rebuild: always
 
+- name: Check if we need to bootstrap an admin
+  no_log: "{{ not (keycloak_bootstrap_logging | default(false)) }}"
+  ansible.builtin.uri:
+    url: "{{ keycloak_url }}/realms/master/protocol/openid-connect/token"
+    method: POST
+    body_format: form-urlencoded
+    body:
+      client_id: "admin-cli"
+      username: "{{ keycloak_local_admin.user }}"
+      password: "{{ keycloak_local_admin.password }}"
+      grant_type: "password"
+  ignore_errors: true
+  register: keycloak_token_response
+
+- name: Set admin required fact
+  ansible.builtin.set_fact:
+    local_admin_user_exists: "{{ keycloak_token_response['status'] == 200 }}"
+
 - name: Bootstrap Keycloak admin
   no_log: "{{ not (keycloak_bootstrap_logging | default(false)) }}"
   community.docker.docker_container:
@@ -53,7 +71,6 @@
       [
         "bootstrap-admin",
         "user",
-        "--username={{ keycloak_bootstrap.admin_user }}",
         "--password:env=KC_BOOTSTRAP_ADMIN_PASSWORD",
         "--optimized",
         "--no-prompt",
@@ -69,6 +86,7 @@
       KC_DB_URL: "jdbc:postgresql://{{ keycloak_db_host }}:{{ keycloak_db_port }}/{{ keycloak_db_name }}"
       KC_DB_USERNAME: "{{ keycloak_db_user }}"
       KC_DB_PASSWORD: "{{ keycloak_db_password }}"
+  when: not local_admin_user_exists
 
 - name: Run Keycloak
   community.docker.docker_container:
@@ -105,6 +123,54 @@
       timeout: 2s
       retries: 15
 
-- ansible.builtin.import_tasks: setup-realm.yml
+# Configure master realm admin
+- name: Create permanent admin user
+  no_log: true
+  community.general.keycloak_user:
+    auth_keycloak_url: "{{ keycloak_url }}"
+    auth_realm: master
+    auth_username: "{{ keycloak_bootstrap.admin_user }}"
+    auth_password: "{{ keycloak_bootstrap.admin_password }}"
+    realm: master
+    username: "{{ keycloak_local_admin.user }}"
+    enabled: true
+    emailVerified: true
+    credentials:
+      - type: password
+        value: "{{ keycloak_local_admin.password }}"
+        temporary: false
+    state: present
+  register: kc_new_admin
+  when: not local_admin_user_exists
+
+- name: Assign admin realm role to permanent user
+  no_log: "{{ not (keycloak_bootstrap_logging | default(false)) }}"
+  community.general.keycloak_user_rolemapping:
+    auth_keycloak_url: "{{ keycloak_url }}"
+    auth_realm: master
+    auth_username: "{{ keycloak_bootstrap.admin_user }}"
+    auth_password: "{{ keycloak_bootstrap.admin_password }}"
+    realm: master
+    uid: "{{ kc_new_admin.end_state.id }}"
+    roles:
+      - name: "admin"
+    state: present
+  when: not local_admin_user_exists
+
+- name: Disable temp-admin bootstrap account
+  no_log: "{{ not (keycloak_bootstrap_logging | default(false)) }}"
+  community.general.keycloak_user:
+    auth_keycloak_url: "{{ keycloak_url }}"
+    auth_realm: "master"
+    auth_username: "{{ keycloak_local_admin.user }}"
+    auth_password: "{{ keycloak_local_admin.password }}"
+    realm: master
+    username: "{{ keycloak_bootstrap.admin_user }}"
+    enabled: false
+    state: present
+  when: not local_admin_user_exists
+
+# Configure custom realm
+- ansible.builtin.import_tasks: setup-target-realm.yml
   vars:
     target_realm: "{{ keycloak_realm.name }}"

infra/ansible/roles/keycloak/tasks/setup-ldap.yml

@@ -4,8 +4,8 @@
     auth_client_id: admin-cli
     auth_keycloak_url: "{{ keycloak_url }}"
     auth_realm: master
-    auth_username: "{{ keycloak_bootstrap.admin_user }}"
-    auth_password: "{{ keycloak_bootstrap.admin_password }}"
+    auth_username: "{{ keycloak_local_admin.user }}"
+    auth_password: "{{ keycloak_local_admin.password }}"
     realm: "{{ target_realm }}"
     name: "STFC LDAP"
     state: present

infra/ansible/roles/keycloak/tasks/setup-realm.yml → infra/ansible/roles/keycloak/tasks/setup-target-realm.yml

@@ -6,8 +6,8 @@
     auth_client_id: admin-cli
     auth_keycloak_url: "{{ keycloak_url }}"
     auth_realm: master
-    auth_username: "{{ keycloak_bootstrap.admin_user }}"
-    auth_password: "{{ keycloak_bootstrap.admin_password }}"
+    auth_username: "{{ keycloak_local_admin.user }}"
+    auth_password: "{{ keycloak_local_admin.password }}"
   when: false
 
 - name: Create Keycloak realm

warehouses/facility_ops/transform/models/staging/estates/stg_electricity_sharepoint_rdm_data.sql

@@ -9,7 +9,7 @@ renamed as (
     select
 
       date_time as power_measured_at,
-      isis_elec_total_power_mwx as total_isis_power_mw
+      isis_elec_total_power_mw as total_isis_power_mw
 
     from source