#!/usr/bin/env python3
#
# build_python_packages.py
#
# Copyright (C) 2021-2022 Franco Masotti (franco \D\o\T masotti {-A-T-} tutanota \D\o\T com)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
r"""build_python_packages.py."""

import contextlib
import multiprocessing
import os
import pathlib
import shlex
import shutil
import signal
import subprocess
import sys

import fpyutils
import yaml
from appdirs import AppDirs


class InvalidCache(Exception):
    pass


class InvalidConfiguration(Exception):
    pass


def check_keys_type(keys: list, key_type) -> bool:
    """Check that all elements of a list correspond to a specific python type."""
    ok = True

    i = 0
    while ok and i < len(keys):
        if isinstance(keys[i], key_type):
            ok = ok & True
        else:
            ok = ok & False
        i += 1

    return ok


def check_values_type(keys: list, values: dict, level_0_value_type, level_1_value_type, has_level_1_value: bool) -> bool:
    """Check that all elements of a list correspond to a specific python type."""
    ok = True

    i = 0
    while ok and i < len(values):
        if isinstance(values[keys[i]], level_0_value_type):
            j = 0
            while has_level_1_value and ok and j < len(values[keys[i]]):
                if isinstance(values[keys[i]][j], level_1_value_type):
                    ok = ok & True
                else:
                    ok = ok & False
                j += 1
        else:
            ok = ok & False

        i += 1

    return ok


##################################
# Check configuration structure  #
##################################
def check_configuration_structure_ignore(configuration: dict) -> bool:
    ok = True
    if ('submodules' in configuration
       and isinstance(configuration['submodules'], dict)
       and 'ignore' in configuration['submodules']
       and isinstance(configuration['submodules']['ignore'], dict)):

        ignore_keys = list(configuration['submodules']['ignore'].keys())

        ok = ok & check_keys_type(ignore_keys, str)
        ok = ok & check_values_type(ignore_keys, configuration['submodules']['ignore'], list, str, True)

    return ok


def check_configuration_structure_base_directory_override(configuration: dict) -> bool:
    ok = True
    if ('submodules' in configuration
       and isinstance(configuration['submodules'], dict)
       and 'base_directory_override' in configuration['submodules']
       and isinstance(configuration['submodules']['base_directory_override'], dict)):

        base_directory_override_keys = list(configuration['submodules']['base_directory_override'].keys())
        ok = ok & check_keys_type(base_directory_override_keys, str)
        ok = ok & check_values_type(base_directory_override_keys, configuration['submodules']['base_directory_override'], str, None, False)
    else:
        ok = False

    return ok


def check_configuration_structure_checkout(configuration: dict) -> bool:
    ok = True
    if ('submodules' in configuration
       and 'checkout' in configuration['submodules']
       and isinstance(configuration['submodules'], dict)
       and isinstance(configuration['submodules']['checkout'], dict)):

        checkout_keys = list(configuration['submodules']['checkout'].keys())
        ok = ok & check_keys_type(checkout_keys, str)
        ok = ok & check_values_type(checkout_keys, configuration['submodules']['checkout'], list, str, True)
    else:
        ok = False

    return ok


def check_configuration_structure_build(configuration: dict) -> bool:
    ok = True
    if ('submodules' in configuration
       and 'build' in configuration['submodules']
       and isinstance(configuration['submodules'], dict)
       and isinstance(configuration['submodules']['build'], dict)
       and 'pre_commands' in configuration['submodules']['build']
       and 'post_commands' in configuration['submodules']['build']):
        pre = configuration['submodules']['build']['pre_commands']
        post = configuration['submodules']['build']['post_commands']
        pre_commands_block_keys = list(pre.keys())
        ok = ok & check_keys_type(pre_commands_block_keys, str)
        post_commands_block_keys = list(post.keys())
        ok = ok & check_keys_type(post_commands_block_keys, str)

        i = 0
        while ok and i < len(pre):
            pre_section = pre[list(pre.keys())[i]]
            ok = ok & check_values_type(list(pre_section.keys()), pre_section, list, str, True)
            i += 1
        i = 0
        while ok and i < len(post):
            post_section = pre[list(post.keys())[i]]
            ok = ok & check_values_type(list(post_section.keys()), post_section, list, str, True)
            i += 1
    else:
        ok = False

    return ok


def check_configuration_structure(configuration: dict) -> bool:
    ok = True
    if ('repository' in configuration
       and 'submodules' in configuration
       and 'other' in configuration
       and 'path' in configuration['repository']
       and 'remote' in configuration['repository']
       and 'default branch' in configuration['repository']
       and isinstance(configuration['submodules'], dict)
       and isinstance(configuration['repository']['path'], str)
       and isinstance(configuration['repository']['remote'], str)
       and isinstance(configuration['repository']['default branch'], str)
       and 'concurrent_workers' in configuration['other']
       and 'unit_workers_per_block' in configuration['other']
       and isinstance(configuration['other']['concurrent_workers'], int)
       and isinstance(configuration['other']['unit_workers_per_block'], int)
       and 'ignored_are_successuful' in configuration['submodules']
       and 'mark_failed_as_successful' in configuration['submodules']
       and isinstance(configuration['submodules']['ignored_are_successuful'], bool)
       and isinstance(configuration['submodules']['mark_failed_as_successful'], bool)):
        ok = True
    else:
        ok = False

    ok = ok & check_configuration_structure_ignore(configuration)

    return ok


def elements_are_unique(struct: list) -> bool:
    unique = False
    if len(struct) == len(set(struct)):
        unique = True

    return unique


#########################
# Check cache structure #
#########################
def check_cache_structure(cache: dict) -> bool:
    ok = True
    if not isinstance(cache, dict):
        ok = False

    elements = list(cache.keys())

    ok = ok & check_keys_type(elements, str)

    # Check that tags are unique within the same git repository.
    i = 0
    while ok and i < len(cache):
        if not elements_are_unique(cache[elements[i]]):
            ok = ok & False
        i += 1

    ok = ok & check_values_type(elements, cache, list, str, True)

    return ok


###########
# Generic #
###########
def send_notification(message: str, notify: dict):
    m = notify['gotify']['message'] + '\n' + message
    if notify['gotify']['enabled']:
        fpyutils.notify.send_gotify_message(
            notify['gotify']['url'],
            notify['gotify']['token'], m,
            notify['gotify']['title'],
            notify['gotify']['priority'])
    if notify['email']['enabled']:
        fpyutils.notify.send_email(message,
                                   notify['email']['smtp server'],
                                   notify['email']['port'],
                                   notify['email']['sender'],
                                   notify['email']['user'],
                                   notify['email']['password'],
                                   notify['email']['receiver'],
                                   notify['email']['subject'])


# See
# https://stackoverflow.com/a/13847807
# CC BY-SA 4.0
# spiralman, bryant1410
@contextlib.contextmanager
def pushd(new_dir):
    previous_dir = os.getcwd()
    os.chdir(new_dir)
    try:
        yield
    finally:
        os.chdir(previous_dir)


def build_message(total_tags: int, total_successful_tags: int) -> str:
    message = '\ncurrent successful package git tags: ' + str(total_successful_tags)
    message += '\ntotal package git tags: ' + str(total_tags)
    if total_tags != 0:
        message += '\ncurrent success rate percent: ' + str((total_successful_tags / total_tags) * 100)
    else:
        message += '\ncurrent success rate percent: 0'

    return message


def print_information(repository_name: str, git_ref: str, message: str = 'processing'):
    print('\n==========================')
    print('note: ' + message + ': ' + repository_name + '; git ref: ' + git_ref)
    print('==========================\n')


#########
# Files #
#########
def read_yaml_file(file: str) -> dict:
    data = dict()
    if pathlib.Path(file).is_file():
        data = yaml.load(open(file, 'r'), Loader=yaml.SafeLoader)

    return data


def read_cache_file(file: str) -> dict:
    cache = read_yaml_file(file)
    if not check_cache_structure(cache):
        raise InvalidCache

    return cache


def update_cache(cache: dict, new_cache: dict):
    for c in cache:
        if c in new_cache:
            cache[c] = list(set(cache[c]).union(set(new_cache[c])))

    for c in new_cache:
        if c not in cache:
            cache[c] = new_cache[c]


def write_cache(cache: dict, new_cache: dict, cache_file: str):
    print('note: writing cache')

    update_cache(cache, new_cache)

    with open(cache_file, 'w') as f:
        f.write(yaml.dump(cache))


#######
# Git #
#######
def git_get_updates(git_executable: str, repository_path: str, repository_remote: str, repository_default_branch: str):
    with pushd(repository_path):
        fpyutils.shell.execute_command_live_output(
            shlex.quote(git_executable)
            + ' pull '
            + repository_remote
            + ' '
            + repository_default_branch
        )
        fpyutils.shell.execute_command_live_output(shlex.quote(git_executable) + ' submodule sync')

        # We might need to add the '--recursive' option for 'git submodule update'
        # to build certain packages. This means that we still depend from external
        # services at build time if we use that option.
        fpyutils.shell.execute_command_live_output(shlex.quote(git_executable) + ' submodule update --init --remote')

        fpyutils.shell.execute_command_live_output(shlex.quote(git_executable) + ' submodule foreach git fetch --tags --force')


def git_remove_untracked_files(git_executable: str):
    fpyutils.shell.execute_command_live_output(shlex.quote(git_executable) + ' checkout --')
    fpyutils.shell.execute_command_live_output(shlex.quote(git_executable) + ' clean -d --force')


def git_get_tags(git_executable: str) -> list:
    r"""Get the list of tags of a git repository.

    :returns: s, a list of tags. In case the git repository has no tags, s is an empty list.
    """
    s = subprocess.run([shlex.quote(git_executable), 'tag'], check=True, capture_output=True)
    s = s.stdout.decode('UTF-8').rstrip().split('\n')

    # Avoid an empty element when there are no tags.
    if s == ['']:
        s = list()

    return s


def git_filter_processed_tags(tags: list, cache: dict, software_name: str) -> tuple:
    r"""Given a list of tags filter out the ones already present in cache."""
    successful_tags: int = 0

    if software_name in cache:
        total_tags_original = len(tags)
        # Filter tags not already processed.
        tags = list(set(tags) - set(cache[software_name]))

        # Tags already in cache must be successful.
        # In case len(tags) < len(cache[software_name]) because configuration changed,
        # pick the minimum between the two.
        successful_tags = min(len(cache[software_name]), total_tags_original)

        # Git repositories without tags: remove cache reference if present.
        if tags == cache[software_name] and tags == ['']:
            tags = []

    return tags, successful_tags


def git_filter_ignore_tags(tags: list, ignore_objects: dict, skip_tags: bool, software_name: str) -> list:
    r"""Given a list of tags filter out the ones in an ignore list."""
    if skip_tags:
        tags = list(set(tags) - set(ignore_objects[software_name]))

    return tags


def git_get_repository_timestamp(git_executable: str) -> str:
    r"""Return the timestamp of the last git ref."""
    return subprocess.run(
        [
            shlex.quote(git_executable),
            'log',
            '-1',
            '--pretty=%ct'
        ], check=True, capture_output=True).stdout.decode('UTF-8').strip()


##########
# Python #
##########
def build_dist(python_executable: str, git_executable: str):
    r"""Build the Python package in a reproducable way.

        Remove all dev, pre-releases, etc information from the package name.
        Use a static timestamp.
        See
        https://github.com/pypa/build/issues/328#issuecomment-877028239
    """
    subprocess.run(
        [
            shlex.quote(python_executable),
            '-m',
            'build',
            '--sdist',
            '--wheel',
            '-C--global-option=egg_info',
            '-C--global-option=--no-date',
            '-C--global-option=--tag-build=',
            '.'
        ], check=True, env=dict(os.environ, SOURCE_DATE_EPOCH=git_get_repository_timestamp(git_executable)))


def upload_dist(twine_executable: str, pypi_url: str, pypi_username: str, pypi_password: str):
    r"""Push the compiled package to a remote PyPI server."""
    subprocess.run(
        [
            shlex.quote(twine_executable),
            'upload',
            '--repository-url',
            pypi_url,
            '--non-interactive',
            '--skip-existing',
            'dist/*'
        ], check=True, env=dict(os.environ, TWINE_PASSWORD=pypi_password, TWINE_USERNAME=pypi_username))


def skip_objects(ignore_objects: dict, software_name: str) -> tuple:
    r"""Determine whether to skip repositories and/or tags."""
    skip_repository = False
    skip_tags = False
    if software_name in ignore_objects:
        if len(ignore_objects[software_name]) == 0:
            skip_repository = True
            skip_tags = False
        else:
            skip_tags = True

    return skip_repository, skip_tags


def set_base_directory_override(base_directory_override: dict, directory: str, software_name: str) -> pathlib.Path:
    r"""Change to the appropriate directory.

    :returns: directory, the path of the directory where the setup files
        are present.

    ..note: Values are reported in the remote configuration
    """
    old_directory = directory
    if software_name in base_directory_override:
        directory = pathlib.Path(directory, base_directory_override[software_name])
        # Check if inner_directory exists.
        # inner_directory usually is equal to absolute_directory
        if not directory.is_dir():
            # Fallback.
            directory = pathlib.Path(old_directory)
    else:
        directory = pathlib.Path(directory)

    return directory


def build_package_pre_post_commands(command_block: dict):
    for block in command_block:
        cmd = list()
        for c in command_block[block]:
            cmd.append(shlex.quote(c))
        try:
            subprocess.run(cmd, check=True)
        except subprocess.CalledProcessError as e:
            print(e)


def build_package(
    git_executable,
    python_executable,
    rm_executable: str,
    twine_executable,
    pypi_url,
    pypi_user,
    pypi_password,
    cache: dict,
    directory: str,
    software_name: str,
    tag: str,
    base_directory_override: dict,
    submodule_mark_failed_as_successful: bool,
    command_block_pre: dict,
    command_block_post: dict,
) -> int:
    r"""Checkout, compile and push.

    This function processes repositories without tags.
    In this case they are marked as successful but are not added to the cache.
    """
    successful_tag: int = 0
    directory_relative_path = directory.stem
    # Cleanup.
    git_remove_untracked_files(git_executable)

    print_information(directory_relative_path, tag, 'processing')

    # Checkout repository with tags: avoid checking out tagless repositories.
    if tag != str():
        fpyutils.shell.execute_command_live_output(
            shlex.quote(git_executable)
            + ' checkout '
            + tag
        )

    # Decide whether to change directory based on remote configuration.
    inner_directory = set_base_directory_override(base_directory_override, directory, software_name)
    subdirectory_absolute_path = str(inner_directory)
    with pushd(subdirectory_absolute_path):
        fpyutils.shell.execute_command_live_output(shlex.quote(rm_executable) + ' -rf build dist')
        try:
            build_package_pre_post_commands(command_block_pre)
            build_dist(python_executable, git_executable)
            build_package_pre_post_commands(command_block_post)

            upload_dist(twine_executable, pypi_url, pypi_user, pypi_password)

            # Register success in cache yaml file.
            successful_tag = 1
            if tag != str():
                cache[software_name].append(tag)

        except subprocess.CalledProcessError:
            print_information(directory.stem, tag, 'error')

            if submodule_mark_failed_as_successful:
                # Do not add an empty git tag to the cache.
                successful_tag = 1
                if tag != str():
                    cache[software_name].append(tag)

        git_remove_untracked_files(git_executable)

    return successful_tag


def read_remote_configuration(repository_path: str) -> dict:
    """Retrieve the configuration of the remote repository."""
    remote_configuration_file = pathlib.Path(repository_path, 'configuration.yaml')
    remote_config = dict()
    if remote_configuration_file.is_file():
        remote_config = yaml.load(open(remote_configuration_file, 'r'), Loader=yaml.SafeLoader)
        if (not check_configuration_structure_base_directory_override(remote_config)
           or not check_configuration_structure_checkout(remote_config)
           or not check_configuration_structure_build(remote_config)):
            raise InvalidConfiguration

    return remote_config


def worker(args: list) -> tuple:
    directory: pathlib.Path = next(args)
    ignore_objects: dict = next(args)
    cache: dict = next(args)
    submodules_ignored_are_successuful: bool = next(args)
    git_executable: str = next(args)
    python_executable: str = next(args)
    rm_executable: str = next(args)
    twine_executable: str = next(args)
    pypi_url: str = next(args)
    pypi_user: str = next(args)
    pypi_password: str = next(args)
    submodule_mark_failed_as_successful: bool = next(args)
    remote_config: dict = next(args)
    repository_path: str = next(args)

    total_tags_including_processed: int = 0
    total_tags_to_process: int = 0
    total_successful_tags_including_processed: int = 0
    successful_tags: int = 0

    # The software name is used in the cache as key.
    software_name = pathlib.Path(directory).stem
    skip_repository, skip_tags = skip_objects(ignore_objects, software_name)
    submodule_absolute_path = str(pathlib.Path(repository_path, directory))

    # Create a new cache slot.
    if software_name not in cache:
        cache[software_name] = list()

    # Mark ignored repositories as successful if the setting is enabled.
    # Save in cache.
    if (skip_repository
       and submodules_ignored_are_successuful):
        with pushd(submodule_absolute_path):
            tags = git_get_tags(git_executable)

            # Bulk append: all tags are successful.
            cache[software_name] = tags

            total_tags: int = len(tags)
            successful_tags = total_tags
            total_tags_including_processed = total_tags
            total_tags_to_process = total_tags

    # Process the repository normally.
    elif not skip_repository:
        with pushd(submodule_absolute_path):
            # Cleanup previous runs.
            git_remove_untracked_files(git_executable)

            # Get all git tags and iterate.
            tags = git_get_tags(git_executable)
            # Useful to detect tagless repositories.
            total_tags_original = len(tags)

            if total_tags_original > 0:
                # Filter out tags that are in the ignore list.
                tags = git_filter_ignore_tags(tags, ignore_objects, skip_tags, software_name)

                # The 'checkout' section in the remote configuration rewrites all tags.
                if software_name in remote_config['submodules']['checkout']:
                    tags = remote_config['submodules']['checkout'][software_name]

                total_tags_including_processed = len(tags)

                # Filter tags not already processed (i.e: not already in cache marked as successful).
                tags, successful_tags = git_filter_processed_tags(tags, cache, software_name)
                total_tags_to_process = len(tags)
            else:
                # Tagless repositories have 1 dummy tag.
                total_tags_to_process = 1
                total_tags_including_processed = 1

            # Get pre-post build commands.
            if software_name in remote_config['submodules']['build']['pre_commands']:
                pre_commands = remote_config['submodules']['build']['pre_commands'][software_name]
            else:
                pre_commands = dict()
            if software_name in remote_config['submodules']['build']['post_commands']:
                post_commands = remote_config['submodules']['build']['post_commands'][software_name]
            else:
                post_commands = dict()

            # Build the Python package.
            if total_tags_original == 0:
                print('note: tag 1 of 1')
                # Tagless repository. Pass an empty string as tag id.
                successful_tags += build_package(
                    git_executable,
                    python_executable,
                    rm_executable,
                    twine_executable,
                    pypi_url,
                    pypi_user,
                    pypi_password,
                    cache,
                    directory,
                    software_name,
                    str(),
                    remote_config['submodules']['base_directory_override'],
                    submodule_mark_failed_as_successful,
                    pre_commands,
                    post_commands,
                )
            else:
                i = 1
                for t in tags:
                    print('note: tag ' + str(i) + ' of ' + str(len(tags)))
                    successful_tags += build_package(
                        git_executable,
                        python_executable,
                        rm_executable,
                        twine_executable,
                        pypi_url,
                        pypi_user,
                        pypi_password,
                        cache,
                        directory,
                        software_name,
                        t,
                        remote_config['submodules']['base_directory_override'],
                        submodule_mark_failed_as_successful,
                        pre_commands,
                        post_commands,
                    )
                    i += 1

    total_successful_tags_including_processed = successful_tags

    return cache, total_tags_including_processed, total_tags_to_process, total_successful_tags_including_processed


def build_worker_arg(
    directory: pathlib.Path,
    ignore_objects: dict,
    cache: dict,
    submodules_ignored_are_successuful: bool,
    git_executable,
    python_executable: str,
    rm_executable: str,
    twine_executable: str,
    pypi_url: str,
    pypi_user: str,
    pypi_password: str,
    submodule_mark_failed_as_successful: str,
    remote_config: dict,
    repository_path: str
) -> iter:

    return iter([directory, ignore_objects, cache, submodules_ignored_are_successuful,
                 git_executable, python_executable, rm_executable, twine_executable, pypi_url,
                 pypi_user, pypi_password, submodule_mark_failed_as_successful, remote_config, repository_path])


def process(
    cache: dict,
    cache_file: str,
    ignore_objects: dict,
    submodules_ignored_are_successuful: bool,
    notify: dict,
    git_executable: str,
    python_executable: str,
    rm_executable: str,
    twine_executable: str,
    pypi_url: str,
    pypi_user: str,
    pypi_password: str,
    repository_path: str,
    repository_remote: str,
    repository_default_branch: str,
    submodule_mark_failed_as_successful: bool,
    concurrent_workers: int = multiprocessing.cpu_count(),
    unit_workers_per_block: int = 1,
):
    quit: bool = False

    # Define and register the signals.
    def signal_handler(*args):
        global quit
        quit = True
        print('\n==========================')
        print('note: signal received. finished queued workers and writing ' + str(len(cache)) + ' repository elements to cache before exit')
        print('==========================\n')
    signal.signal(signal.SIGINT, signal_handler)
    signal.signal(signal.SIGTERM, signal_handler)

    # Autodetect.
    if concurrent_workers <= 0:
        concurrent_workers = multiprocessing.cpu_count()
    if unit_workers_per_block <= 0:
        unit_workers_per_block = 1

    git_get_updates(git_executable, repository_path, repository_remote, repository_default_branch)

    remote_config = read_remote_configuration(repository_path)

    # Go to the submodules subdirectory.
    repository_path = pathlib.Path(repository_path, 'submodules')

    dirs: list = [x for x in pathlib.Path(repository_path).iterdir()]
    len_dirs: int = len(dirs)
    n: int = 0
    new_cache: dict = dict()
    new_total_tags_including_processed: int = 0
    new_total_tags_to_process: int = 0
    new_total_successful_tags: int = 0
    import copy
    tmp_cache: dict = copy.deepcopy(cache)

    while n < len_dirs and not quit:
        signal.signal(signal.SIGINT, signal.SIG_IGN)
        signal.signal(signal.SIGTERM, signal.SIG_IGN)
        signal.signal(signal.SIGABRT, signal.SIG_IGN)
        signal.signal(signal.SIGALRM, signal.SIG_IGN)

        remaining_dirs: int = len_dirs - n
        args: list = list()
        step = min(concurrent_workers * unit_workers_per_block, remaining_dirs)

        print('note: remaining ' + str(remaining_dirs) + ' repositories')

        for i in range(0, step):
            # Skip non.directories and the ./.git directory.
            if dirs[n + i].is_dir() and dirs[n + i].stem != '.git':
                args.append(build_worker_arg(dirs[n + i], ignore_objects, tmp_cache,
                            submodules_ignored_are_successuful, git_executable,
                            python_executable, rm_executable, twine_executable, pypi_url,
                            pypi_user, pypi_password, submodule_mark_failed_as_successful,
                            remote_config, repository_path))

        pool = multiprocessing.Pool(processes=concurrent_workers)
        try:
            signal.signal(signal.SIGINT, signal_handler)
            signal.signal(signal.SIGTERM, signal_handler)
            signal.signal(signal.SIGABRT, signal_handler)
            signal.signal(signal.SIGALRM, signal_handler)
            result = pool.map_async(worker, args)
            rr = result.get(timeout=3600)
            for r in rr:
                update_cache(new_cache, r[0])
                new_total_tags_including_processed += r[1]
                new_total_tags_to_process += r[2]
                new_total_successful_tags += r[3]
        except (KeyboardInterrupt, InterruptedError):
            pool.terminate()
            pool.join()
            quit = True
        else:
            pool.close()
            pool.join()

        n += step

    if pool:
        # Cleanup.
        pool.close()
        pool.join()

    write_cache(cache, new_cache, cache_file)

    message = build_message(new_total_tags_including_processed, new_total_successful_tags)
    print(message)
    send_notification(message, notify)


if __name__ == '__main__':
    def main():
        configuration_file = shlex.quote(sys.argv[1])
        config = yaml.load(open(configuration_file, 'r'), Loader=yaml.SafeLoader)
        if not check_configuration_structure(config):
            raise InvalidConfiguration

        dirs = AppDirs('build_pypi_packages')
        # Read the cache file.
        if config['files']['cache']['clear']:
            shutil.rmtree(dirs.user_cache_dir, ignore_errors=True)
        pathlib.Path(dirs.user_cache_dir).mkdir(mode=0o700, exist_ok=True, parents=True)
        cache_file = str(pathlib.Path(dirs.user_cache_dir, config['files']['cache']['file']))
        cache = read_cache_file(cache_file)

        process(
            cache,
            cache_file,
            config['submodules']['ignore'],
            config['submodules']['ignored_are_successuful'],
            config['notify'],
            config['files']['executables']['git'],
            config['files']['executables']['python'],
            config['files']['executables']['rm'],
            config['files']['executables']['twine'],
            config['pypi']['url'],
            config['pypi']['username'],
            config['pypi']['password'],
            config['repository']['path'],
            config['repository']['remote'],
            config['repository']['default branch'],
            config['submodules']['mark_failed_as_successful'],
            config['other']['concurrent_workers'],
            config['other']['unit_workers_per_block'],
        )

    main()