Attempted to fix issue where the vault tries to re-initialized even though it's already initialized (because of PV backed storage) + uncommented some of the CI/CD automation (as become more comforatable with it

.forgejo/workflows/deploy.yml

@@ -14,86 +14,86 @@ jobs:
         uses: actions/checkout@v4
 
       # Build a new container image from the code
-      #- name: Update Image
+      - name: Update Image
-      #  run: |
+        run: |
-      #    # Parse information from the metadata.yaml file
+          # Parse information from the metadata.yaml file
-      #    IMAGE_NAME=$(yq '.name' metadata.yaml)
+          IMAGE_NAME=$(yq '.name' metadata.yaml)
-      #    IMAGE_NAME=${IMAGE_NAME#\"}  # Remove leading quote
+          IMAGE_NAME=${IMAGE_NAME#\"}  # Remove leading quote
-      #    IMAGE_NAME=${IMAGE_NAME%\"}  # Remove trailing quote
+          IMAGE_NAME=${IMAGE_NAME%\"}  # Remove trailing quote
-      #    echo "Image Name: $IMAGE_NAME"
+          echo "Image Name: $IMAGE_NAME"
-      #    echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV
+          echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV
-      #
+      
-      #    LOCAL_VERSION=$(yq '.version' metadata.yaml)
+          LOCAL_VERSION=$(yq '.version' metadata.yaml)
-      #    LOCAL_VERSION=${LOCAL_VERSION#\"}  # Remove leading quote
+          LOCAL_VERSION=${LOCAL_VERSION#\"}  # Remove leading quote
-      #    LOCAL_VERSION=${LOCAL_VERSION%\"}  # Remove trailing quote
+          LOCAL_VERSION=${LOCAL_VERSION%\"}  # Remove trailing quote
-      #    echo "Image Local Version: $LOCAL_VERSION"
+          echo "Image Local Version: $LOCAL_VERSION"
-      #
+      
-      #    REMOTE_TAGS_WORK_OUTPUT=$(skopeo list-tags docker://${{ secrets.REPOSITORY_HOSTNAME }}/k8s/$IMAGE_NAME 2>/dev/null || echo "")
+          REMOTE_TAGS_WORK_OUTPUT=$(skopeo list-tags docker://${{ secrets.REPOSITORY_HOSTNAME }}/k8s/$IMAGE_NAME 2>/dev/null || echo "")
-      #    if [ -n "$REMOTE_TAGS_WORK_OUTPUT" ]; then
+          if [ -n "$REMOTE_TAGS_WORK_OUTPUT" ]; then
-      #      IFS=' ' read -r -a REMOTE_TAGS <<< $(skopeo list-tags docker://${{ secrets.REPOSITORY_HOSTNAME }}/k8s/$IMAGE_NAME | jq -r '.Tags | @sh')
+            IFS=' ' read -r -a REMOTE_TAGS <<< $(skopeo list-tags docker://${{ secrets.REPOSITORY_HOSTNAME }}/k8s/$IMAGE_NAME | jq -r '.Tags | @sh')
-      #    else
+          else
-      #      echo "Image not found in the repository. Will need to add it..."
+            echo "Image not found in the repository. Will need to add it..."
-      #      
+            
-      #      # Set a blank value so that it WON'T match the local version
+            # Set a blank value so that it WON'T match the local version
-      #      IFS=' ' read -r -a REMOTE_TAGS <<< ""
+            IFS=' ' read -r -a REMOTE_TAGS <<< ""
-      #    fi
+          fi
-      #
+      
-      #    echo "Remote Tags (number: ${#REMOTE_TAGS[@]}): ${REMOTE_TAGS[@]}"
+          echo "Remote Tags (number: ${#REMOTE_TAGS[@]}): ${REMOTE_TAGS[@]}"
-      #
+      
-      #    has_match='false'
+          has_match='false'
-      #    if [ ${#REMOTE_TAGS[@]} -gt 0 ]; then
+          if [ ${#REMOTE_TAGS[@]} -gt 0 ]; then
-      #      # Loop through the remote tags and check if any of them match the local version
+            # Loop through the remote tags and check if any of them match the local version
-      #      for REMOTE_TAG in ${REMOTE_TAGS[@]}; do
+            for REMOTE_TAG in ${REMOTE_TAGS[@]}; do
-      #        REMOTE_TAG=${REMOTE_TAG#\'}  # Remove leading quote
+              REMOTE_TAG=${REMOTE_TAG#\'}  # Remove leading quote
-      #        REMOTE_TAG=${REMOTE_TAG%\'}  # Remove trailing quote
+              REMOTE_TAG=${REMOTE_TAG%\'}  # Remove trailing quote
-      #
+      
-      #        # Check if the remote tag is the same as the local tag
+              # Check if the remote tag is the same as the local tag
-      #        if [ "$REMOTE_TAG" == "v$LOCAL_VERSION" ]; then
+              if [ "$REMOTE_TAG" == "v$LOCAL_VERSION" ]; then
-      #          echo "Remote version matches local version!"
+                echo "Remote version matches local version!"
-      #          has_match='true'
+                has_match='true'
-      #          break
+                break
-      #        fi
+              fi
-      #      done
+            done
-      #    fi
+          fi
-      #    
+          
-      #    # If a remote tag that matches the local version already exists, increment the local version's patch version
+          # If a remote tag that matches the local version already exists, increment the local version's patch version
-      #    if [ "$has_match" == 'true' ]; then
+          if [ "$has_match" == 'true' ]; then
-      #      echo "Because the remote version matches the local version, we need to increment the local version's patch number."
+            echo "Because the remote version matches the local version, we need to increment the local version's patch number."
-      #      
+            
-      #      # Increment the patch version of the local version (Ex. 1.0.0 -> 1.0.1)
+            # Increment the patch version of the local version (Ex. 1.0.0 -> 1.0.1)
             #IFS='.' read -r major minor patch <<< "$LOCAL_VERSION"
             #patch=$((patch + 1))
             #NEW_LOCAL_VERSION="$major.$minor.$patch"
-      #      
+             
             #echo "New Local Version: $NEW_LOCAL_VERSION"
             #echo "Committing container version change..."
-      #
+          
             #sed -i "s|version: $LOCAL_VERSION|version: $NEW_LOCAL_VERSION|g" metadata.yaml
-      #
+          
             #LOCAL_VERSION=$NEW_LOCAL_VERSION
-      #
+          
-      #      # Update remote URL to use the GITHUB_TOKEN for authentication
+            # Update remote URL to use the GITHUB_TOKEN for authentication
-      #      git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git
+            #git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@git.bridgemanaccessible.ca/${{ github.repository }}.git
-      #
+          
-      #      # Setup git user details for committing the version change and tag
+            # Setup git user details for committing the version change and tag
-      #      git config user.name "GitHub Actions"
+            #git config user.name "Forgejo Actions"
-      #      git config user.email "actions@github.com"
+            #git config user.email "actions@git.bridgemanaccessible.ca"
-      #
+          
-      #      # Commit the version change to the `package.json` file
+            # Commit the version change to the `package.json` file
             #git add metadata.yaml
-      #      git commit -m "[Github Actions] Update container version to $(yq -r '.version' metadata.yaml)"
+            #git commit -m "[Forgejo Actions] Update container version to $(yq -r '.version' metadata.yaml)"
-      #
+          
-      #      # Push the changes to the repository
+            # Push the changes to the repository
             #git push origin HEAD:main
-      #    fi
+          fi
-      #
+      
-      #    # Build and push the init container image to the repository
+          # Build and push the init container image to the repository
-      #    docker build -t ${{ secrets.REPOSITORY_HOSTNAME }}/k8s/$IMAGE_NAME:v$LOCAL_VERSION .
+          docker build -t ${{ secrets.REPOSITORY_HOSTNAME }}/k8s/$IMAGE_NAME:v$LOCAL_VERSION .
-      #    docker push ${{ secrets.REPOSITORY_HOSTNAME }}/k8s/$IMAGE_NAME:v$LOCAL_VERSION
+          docker push ${{ secrets.REPOSITORY_HOSTNAME }}/k8s/$IMAGE_NAME:v$LOCAL_VERSION
-      #
+      
-      #    # Note, this is the version NOT the tag
+          # Note, this is the version NOT the tag
-      #    # This is because the `update-k8s-deployment-image` script automatically prepends the `v` to the version
+          # This is because the `update-k8s-deployment-image` script automatically prepends the `v` to the version
-      #    echo "CONTAINER_IMAGE_VERSION=$LOCAL_VERSION" >> $GITHUB_ENV
+          echo "CONTAINER_IMAGE_VERSION=$LOCAL_VERSION" >> $GITHUB_ENV
       
       #- name: Update vault images in various deployments
       #  run: |

.gitignore

@@ -1,2 +1,5 @@
 # Environment Variables
 .env
+
+# Ignore VSCode settings (that don't need to be shared)
+.vscode/

metadata.yaml

@@ -3,6 +3,6 @@
 # We decided to use a YAML file (instead of JSON as an example) because it allows the use of comments, which can be helpful for documentation/explanation purposes like this.
 
 name: hashicorp-vault
-version: 1.0.14
+version: 1.0.15
 description: Customized implementation of the Hashicorp Vault image.
 maintainer: Bridgeman Accessible <info@bridgemanaccessible.ca>

setup-scripts/app-role-access.py

@@ -12,7 +12,7 @@
 # | Any modification or use of this script is at the user's own risk. |
 # |*******************************************************************|
 
-import os, sys, subprocess, logging
+import os, sys, subprocess, json, logging
 
 from CommandRunner import CommandRunner
 
@@ -24,6 +24,54 @@ from CommandRunner import CommandRunner
 #        # Log the error and re-raise or handle as appropriate
 #        raise RuntimeError(f"Command '{' '.join(command)}' failed with error: {e.stderr}") from e
 
+def check_app_role_enabled():
+    """Check if the AppRole auth method is enabled in vault
+
+    Returns:
+        bool: True if AppRole is enabled, False otherwise
+    """
+
+    # List the enabled auth methods
+    auth_return_code, auth_output, auth_err = CommandRunner.run_command('vault auth list -format=json')
+
+    # If non-zero return code, raise an error
+    if auth_return_code != 0:
+        logging.error('Failed to list auth methods')
+        logging.error('Auth Output: ' + auth_output)
+        logging.error('Auth Error: ' + auth_err)
+        raise RuntimeError('Failed to list auth methods')
+    
+    # Parse the output
+    auth_methods = json.loads(auth_output)
+
+    # Use the existence of the 'approle/' key to determine if AppRole is enabled
+    return 'approle/' in auth_methods
+
+def check_policy_exists(policy_name: str) -> bool:
+    """Check if a policy exists in vault
+
+    Args:
+        policy_name (str): The name of the policy to check
+    Returns:
+        bool: True if the policy exists, False otherwise
+    """
+    
+    # List the policies
+    policy_return_code, policy_output, policy_err = CommandRunner.run_command('vault policy list -format=json')
+
+    # If non-zero return code, raise an error
+    if policy_return_code != 0:
+        logging.error('Failed to list policies')
+        logging.error('Policy Output: ' + policy_output)
+        logging.error('Policy Error: ' + policy_err)
+        raise RuntimeError('Failed to list policies')
+    
+    # Parse the output
+    policies = json.loads(policy_output)
+
+    # Check if the policy name exists in the returned list
+    return policy_name in policies
+
 def create_policy(policy_name: str, policy_capabilities: list[str], policy_path: str = 'secret/*'):
     """Create a policy in vault
 
@@ -33,16 +81,49 @@ def create_policy(policy_name: str, policy_capabilities: list[str], policy_path:
         policy_path (str, optional): The path the policy should apply to. Defaults to all secrets (`secret/*`).
     """
 
+    # Create the "policy file" that is specified on the command line
     policy_caps = '["' + '","'.join(policy_capabilities) + '"]'
     policy_content = 'path "' + policy_path + '" {\n  capabilities = ' + policy_caps + '\n}'
     policy = '<<EOF\n' + policy_content + '\nEOF'
+
+    # Write the policy to vault
     policy_return_code, policy_output, policy_err = CommandRunner.run_command('vault policy write ' + policy_name + ' - ' + policy, False)
-    if policy_return_code == 2:
+    
+    # If non-zero return code, raise an error
+    if policy_return_code != 0:
         logging.error('Failed to create the policy')
         logging.error('Policy Output: ' + policy_output)
         logging.error('Policy Error: ' + policy_err)
         raise RuntimeError('Failed to create the policy')
 
+def check_app_role_exists(role_name: str) -> bool:
+    """Check if an AppRole role exists in vault
+
+    Args:
+        role_name (str): The name of the role to check
+    Returns:
+        bool: True if the role exists, False otherwise
+    """
+    
+    # To use `vault list`, we need to specify the "path" (which is a slash separated string dictating the hierarchy to the thing we want to list)
+    role_list_path = '/'.join(['auth', 'approle', 'role'])
+    
+    # List the roles
+    role_return_code, role_output, role_err = CommandRunner.run_command(f'vault list --format=json {role_list_path}')
+
+    # If non-zero return code, raise an error
+    if role_return_code != 0:
+        logging.error('Failed to list AppRole roles')
+        logging.error('Role Output: ' + role_output)
+        logging.error('Role Error: ' + role_err)
+        raise RuntimeError('Failed to list AppRole roles')
+    
+    # Parse the output
+    roles = json.loads(role_output)
+
+    # Check if the role name exists in the returned list
+    return role_name in roles
+
 def get_role_id(role_name: str) -> str:
     """Get the `role_id` for a given role
 
@@ -53,9 +134,22 @@ def get_role_id(role_name: str) -> str:
         str: The `role_id` for the given role
     """
 
-    # Get the role_id from vault
+    # The read path for the role_id of the given role (path being a slash separated string dictating the hierarchy to the thing we want to read)
     role_read_path = '/'.join(['auth', 'approle', 'role', role_name, 'role-id'])
-    role_return_code, role_id, role_err = CommandRunner.run_command('vault read ' + role_read_path + ' | grep role_id | awk \'{print $2}\'')
+    
+    # Get the role_id from vault
+    role_return_code, role_id_output, role_id_err = CommandRunner.run_command('vault read --format=json ' + role_read_path)
+
+    # If non-zero return code, raise an error
+    if role_return_code != 0:
+        logging.error('Failed to get the role_id for role: ' + role_name)
+        logging.error('Role ID Output: ' + role_id_output)
+        logging.error('Role ID Error: ' + role_id_err)
+        raise RuntimeError('Failed to get the role_id for role: ' + role_name)
+    
+    # Parse the role_id from the output
+    role_id_json = json.loads(role_id_output)
+    role_id = role_id_json['data']['role_id']
 
     logging.debug('Role ID: ' + role_id)
 
@@ -71,9 +165,22 @@ def get_secret_id(role_name: str) -> str:
         str: The `secret_id` for the given role
     """
 
-    # Get the secret_id from vault
+    # The write path for the secret_id of the given role (path being a slash separated string dictating the hierarchy to the thing we want to write)
     secret_write_path = '/'.join(['auth', 'approle', 'role', role_name, 'secret-id'])
-    secret_return_code, secret_id, secret_err = CommandRunner.run_command('vault write -f ' + secret_write_path + ' | grep "secret_id " | awk \'{print $2}\'')
+    
+    # Get the secret_id from vault (by writing to the secret-id endpoint)
+    secret_return_code, secret_id_output, secret_id_err = CommandRunner.run_command('vault write --format=json -f ' + secret_write_path)
+    
+    # If non-zero return code, raise an error
+    if secret_return_code != 0:
+        logging.error('Failed to get the secret_id for role: ' + role_name)
+        logging.error('Secret ID Output: ' + secret_id_output)
+        logging.error('Secret ID Error: ' + secret_id_err)
+        raise RuntimeError('Failed to get the secret_id for role: ' + role_name)
+
+    # Parse the secret_id from the output
+    secret_id_json = json.loads(secret_id_output)
+    secret_id = secret_id_json['data']['secret_id']
 
     logging.debug('Secret ID: ' + secret_id)
 
@@ -90,10 +197,19 @@ def create_app_role(role_name: str, policy_name: str) -> tuple[str, str]:
         tuple[str, str]: The `role_id` and `secret_id` of the newly created role
     """
 
-    # Create a role
+    # The write path for creating the role (path being a slash separated string dictating the hierarchy to the thing we want to write)
     role_write_path = '/'.join(['auth', 'approle', 'role', role_name])
+
+    # Create a role
     role_write_return_code, role_write_output, role_write_err = CommandRunner.run_command('vault write ' + role_write_path + ' token_policies="' + policy_name + '"')
     
+    # If non-zero return code, raise an error
+    if role_write_return_code != 0:
+        logging.error('Failed to create AppRole role: ' + role_name)
+        logging.error('Role Write Output: ' + role_write_output)
+        logging.error('Role Write Error: ' + role_write_err)
+        raise RuntimeError('Failed to create AppRole role: ' + role_name)
+
     logging.debug(role_write_output)
 
     # Get the role_id
@@ -165,6 +281,7 @@ def main(token: str):
     # Login to vault
     CommandRunner.run_command(f'vault login {token}')
 
+    if not check_app_role_enabled():
         # Enable approle auth method
         CommandRunner.run_command('vault auth enable approle')
     
@@ -179,6 +296,7 @@ def main(token: str):
 
     logging.debug(f'Policy Name: {policy_name}')
     
+    if not check_policy_exists(policy_name):
         # The policy capabilities can be set via the `POLICY_CAPABILITIES` environment variable
         # Or defaults to `['read', 'create', 'update']` if not set
         if 'POLICY_CAPABILITIES' in os.environ:
@@ -192,12 +310,15 @@ def main(token: str):
 
     # -- create an approle role --
 
-    # Create a role
+    # The role name can be set via the `ROLE_NAME` environment variable
+    # Or defaults to `node-app` if not set
     if 'ROLE_NAME' in os.environ:
         role_name = os.environ['ROLE_NAME']
     else:
         role_name = 'node-app'
     
+    if not check_app_role_exists(role_name):
+        # Create a role
         role_id, secret_id = create_app_role(role_name, policy_name)
 
         # Save the role_id and secret_id to a backup file
@@ -205,7 +326,20 @@ def main(token: str):
 
         # Save the role_id and secret_id to a file
         save_role_vars_to_file(role_id, secret_id)
+    else:
+        # Get the existing role_id and secret_id
+        role_id = get_role_id(role_name)
+        secret_id = get_secret_id(role_name)
+
+        # Save the role_id and secret_id to a file
+        # QUESTION: Should this be conditional on if the file already exists or not?
+        save_role_vars_to_file(role_id, secret_id)
+    
+    logging.info('AppRole setup complete')
 
 if __name__ == '__main__':
+    # Parse the root token (used to login) from the command line arguments
     token = sys.argv[1]
+
+    # Run the script's main function
     main(token)

setup-scripts/prod-setup.py

@@ -2,6 +2,46 @@ import os, json
 from CommandRunner import CommandRunner
 
 class Initializer:
+    def check_if_initialized(self) -> bool:
+        """Check if the vault is already initialized
+
+        Returns:
+            bool: If the vault is initialized or not
+        """
+
+        # Get the status of the vault
+        # Note, because it returns a non-zero exit code when the vault is uninitialized, we set check to False
+        # Which is also why we need to check the return code manually
+        init_status_returncode, init_status_raw, init_status_err = CommandRunner.run_command('vault status -format=json', False)
+
+        # Verify the return code
+        # Note, because there can be other meanings for return (ex. 2 for Sealed), we don't use a hard check here.
+        if init_status_returncode != 0 and init_status_returncode != 2:
+            print(f'[WARNING] Status Return Code (for Init Check): {init_status_returncode}')
+            #raise RuntimeError('Failed to get the initialization status of the vault')
+        
+        # Print the raw status
+        #print(init_status_raw)
+        
+        # Parse the initialized stat from the status
+        init_status = json.loads(init_status_raw)['initialized']
+
+        print(f'Is Initialized: {init_status}')
+
+        return init_status
+    
+    def check_root_token_and_unseal_keys_files_exist(self) -> bool:
+        """Check if the root token and unseal keys files exist
+
+        Returns:
+            bool: If both files exist
+        """
+
+        root_token_exists = os.path.exists('/vault/creds/root-token')
+        unseal_keys_exists = os.path.exists('/vault/creds/unseal-keys')
+
+        return root_token_exists and unseal_keys_exists
+
     def create_unseal_keys_file(self, file = '/vault/creds/unseal-keys'):
         """Write the vault's unseal keys to a file
 
@@ -46,11 +86,11 @@ class Initializer:
         self.create_unseal_keys_file()
         self.create_root_token_file()
 
-    def is_vault_unsealed(self) -> bool:
+    def is_vault_sealed(self) -> bool:
         """Check if the vault is sealed or not
 
         Returns:
-            bool: If the vault is unsealed or not
+            bool: If the vault is sealed or not
         """
 
         # Get the status of the vault
@@ -63,7 +103,7 @@ class Initializer:
             raise RuntimeError('Failed to get the status of the vault')
         
         # Print the raw status
-        print(seal_status_raw)
+        #print(seal_status_raw)
         
         # Parse the seal stat from the status
         seal_status = json.loads(seal_status_raw)['sealed']
@@ -128,17 +168,35 @@ class Initializer:
 
 def main():
     initializer = Initializer()
-    # Check if the root-token file and unseal keys files exist
+
-    #if os.path.exists('/vault/creds/root-token') and os.path.exists('/vault/creds/unseal-keys'):
+    # We only want to initialize the vault if it isn't initialized already
-    if not initializer.is_vault_unsealed():
+    # Because we use Persistent Volumes (PVs) for vault data on restarts the vault is already initialized
-        print('Vault already setup. Skipping...')
+    if not initializer.check_if_initialized():
-        # QUESTION: Should there be code here to get the Role ID and Secret ID in case the originally created .env file doesn't exist for some reason 
-    else:
         initializer.init_vault()
+        
+        # This is just a safety check/measure to ensure the script can continue
+        # The only time this would likely be triggered is if there was some kind of PV desyncronization but it shouldn't really happen.
+        if not initializer.check_root_token_and_unseal_keys_files_exist():
+            raise RuntimeError('Vault is in an inconsistent state for this script to continue. Please ensure the vault can be initialized OR both the root token and unseal keys files exist alongside and initialized vault.')
+
+        # Check if the vault is sealed (as we need to unseal it to set it up)
+        if initializer.is_vault_sealed():
             initializer.unseal_vault()
+        else:
+            print('Vault is already unsealed. Proceeding to setup...')
+
+        # QUESTION: If the vault data is already setup (PV on restart) do we need to re-setup this stuff?
         initializer.setup_secrets_engine()
         initializer.setup_audit_device()
         initializer.setup_app_role_access()
+    else:
+        print('Vault is already initialized. Skipping initialization and setup...')
+
+        # Check if the vault is already unsealed (we assume it's already setup properly if it is)
+        if initializer.is_vault_sealed():
+            initializer.unseal_vault()
+        
+        # Code goes here to get the Role ID and Secret ID (app role access) in the case that the originally created .env file doesn't exist anymore for some reason 
 
 if __name__ == '__main__':
     main()