# Copyright 2015-2020 VMware, Inc.  All rights reserved. -- VMware Confidential
#
'''
Script responsible to discover all components which should take part of upgrade
workflow, validates components metadata, validates inter-component dependencies
and persist the component discovery result for later patch phases.
'''
import os
import logging
import json
import shutil

from math import ceil

from extensions import Hook
from patch_specs import PatchContext, DiscoveryResult, RequirementsResult, Mismatch
from patch_errors import UserError, ComponentError
from os_utils import createDirectory
import transport
from status_reporting_sdk.json_utils import ObjectJsonConverter, prettyJsonDump
from rpm_utils import validateRpmsRemoval, getInsufficientStorageForRpmInstallation
from l10n import msgMetadata as _T, localizedString as _

from vmware_b2b.patching.utils.resource_utils import getPayloadFilePath
from vmware_b2b.patching.data.model import Component, PatchPhaseContext
from vmware_b2b.patching.executor.execution_facade import executeHook
from vmware_b2b.patching.utils.reporting_utils import INTERNAL_ERROR_TEXT, INTERNAL_ERROR_RESOLUTION, \
    configureLocalization

logger = logging.getLogger(__name__)

CANNOT_REMOVE_RPM_WARNING_TEXT = _T('ur.rpm.removal.warning.text',
                           "It won't be possible to remove deprecated rpms %s from the system")
CANNOT_REMOVE_RPM_WARNING_DESCRIPTION = _(_T('ur.rpm.removal.warning.description',
                                 "As part of the patch deprecated products rpms need to be removed"
                                 " from the system, however this won't be possible due"
                                 " to dependency problems."))
CANNOT_REMOVE_RPM_ERROR_RESOLUTION = _(_T('ur.rpm.removal.warning.resolution',
                                 ' To ensure that there will be no problems with subsequence patches'
                                 ' remove those rpms manually.'))

NOT_ENOUGH_STORAGE_TO_INSTALL_RPM_ERROR_TEXT = _T('patch.rpm.no.storage.install.error.text',
                                 "Insufficient storage space on partition \"%s\".")
NOT_ENOUGH_STORAGE_TO_INSTALL_RPM_ERROR_DESCRIPTION = _(_T('patch.rpm.no.storage.install.error.description',
                                 "As part of the patch product rpms need to be installed"
                                 " on the system, however this won't be possible due"
                                 " to insufficient storage."))
NOT_ENOUGH_STORAGE_TO_INSTALL_RPM_ERROR_RESOLUTION = _T('patch.rpm.no.storage.install.error.resolution',
                                 'Ensure that on partition \"%s\" there is at least %s MB free.')

CANNOT_PATCH_ON_VC_CONFIGURED_WITH_VCHA_ERROR_TEXT = _(_T("cannot.patch.vcsa.in.vcha.cluster.error.text",
                                "You can not patch a vCenter Server appliance in a vCenter HA cluster."))
CANNOT_PATCH_ON_VC_CONFIGURED_WITH_VCHA_ERROR_RESOLUTION = _(_T("cannot.patch.vcsa.in.vcha.cluster.error.resolution",
                                "You must remove the vCenter HA configuration, apply patches to vCenter Server appliance, and then reconfigure your vCenter HA deployment."))

VCHA_AWARE_FILE = "/etc/vmware-vcha/vcha.aware"

PRECHECKS_FILE = "prechecks.json"

def _isVCHAConfigured():
    """
    This check is added to block B2B/Patching  when vCHA is configured on the
    given vCenter Server.

    This check is based on  the following logic:
    /etc/vmware-vcha/vcha.aware file will be present on vCenter Server when
    a. vCHA configured and vCHA enabled
    b. vCHA configured and vCHA disabled.

    When vCHA is not configured or unconfigured then
    /etc/vmware-vcha/vcha.aware file will not be present

    @return: True if the File exists and False otherwise
    @rtype: bool
    """
    if os.path.exists(VCHA_AWARE_FILE):
        return True
    return False


def _cleanupRootStageDir(stageDir):
    '''Cleans up the data persisted in previous patch process execution.

    @param stageDir: global CPO stage directory. All components stage
      directories will be created as sub-directories of global stage directory.
    @type stageDir: str
    '''
    if createDirectory(stageDir):
        logger.info("Create directory %s", stageDir)
    else:
        logger.info("Cleaning up the data persisted in previous patch process "
                    "execution persisted in %s.", stageDir)
        for f in os.listdir(stageDir):
            transport.remove(transport.local.LocalOperationsManager(),
                             os.path.join(stageDir, f))

def _createComponentModel(name, patchScript, stageDir, discoveryResult,
                          requirementsResult):
    '''Creates a component which wants to be part of the patching process.

    @param name: Component internal name. Component name will be used as
      identificator for distinguishing between components. The name is formed
      from the directory where its patching logic reside. For example:
      /foo/bar/patching/sso -> sso
    @type name: str

    @param patchScript: Holds information about where the patching component
      logic reside. Every component patch hook should be defined at
      <patchScriptPath>/__init__.py
    @type patchScript: str

    @param stageDir: Component specific stage directory. This is a unique
      per component directory, where a component can share data between
      the patching hooks.
    @type stageDir: str

    @param discoveryResult: Information about the component found after
      calling its discovery hook.
    @type discoveryResult: patch_specs.DiscoveryResult

    @param requirementsResult: Information about the component requirements.
    @type requirementsResult: patch_specs.RequirementsResult

    @return: Created component
    @rtype: vmware.patching.data.model.Component
    '''
    requirementsResult = requirementsResult or RequirementsResult()
    return Component(name, patchScript, stageDir, discoveryResult, requirementsResult)

def _discoverComponents(path, stageDir, componentNames, locale, skipPrechecksIds, upgradeType):
    '''Discovers the vCSA components which should take part of the patching
    process by calling their Discovery extension point. The method returns all
    discovered components even those that don't want to participate in the
    patching workflow. If a component wants to take part in the patching workflow,
    then its requirements will be collected(call its requirements hook) as part
    of discovery process.

    @param path: A path to directory containing components scripts files.
    @type path: str

    @param stageDir: Component specific stage directory. This is a unique
      per component directory, where a component can share data between
      the patching hooks.
    @type stageDir: str

    @param componentNames: The component names which the callee is interested in,
      the other will be filtered out. If None is passed, then all components
      will be returned. The name of the component is equal to the directory where
      its patching logic reside.
    @param componentNames: array of strings

    @param locale: Current locale. All user-facing messages should be
      translated to that locale.
    @type locale: str

    @param skipPrechecksIds: List of ids of prechecks to be skipped.
    @type skipPrechecksIds: list(str)

    @param upgradeType: The type of the upgrade being performed. The allowed values
      are either PatchContext.DISRUPTIVE_UPGRADE or PatchContext.NONDISRUPTIVE_UPGRADE.

    @return: All the discovered components.
    @rtype: List of vmware.patching.data.model.Component
    '''
    logger.info('Processing patch directory %s',
                os.path.abspath(os.path.join(path, os.pardir)))

    result = []
    for componentName in os.listdir(path):
        filePath = os.path.join(path, componentName)
        if not os.path.isdir(filePath):
            logger.warning("Invalid patching structure: %s", filePath)
            continue

        if componentNames and componentName not in componentNames:
            logger.info("Skip processing component %s", componentName)
            continue

        try:
            logger.info('Processing patch component %s', filePath)

            compStageDir = os.path.join(stageDir, componentName)
            logger.info("Create component '%s' stage directory %s", componentName,
                        compStageDir)
            createDirectory(compStageDir)

            compContext = PatchContext(compStageDir,
                                       locale,
                                       skipPrechecksIds=skipPrechecksIds,
                                       upgradeType=upgradeType)
            # Cannot use the reporting one as there is no component yet
            discoveryResult = executeHook(filePath, Hook.Discovery, compContext, None,
                                          expectedResultType=(type(None), DiscoveryResult))
            if discoveryResult is None:
                logger.warning("The component %s does not provide "
                               "component metadata, hence won't be patched",
                               componentName)
                continue

            logger.info("Found valid component %s", componentName)

            requirementsResult = None
            if discoveryResult.patchable:
                # Retrieve component requirements only for patchable components.
                # Cannot use the reporting one as there is no component yet.
                requirementsResult = executeHook(filePath, Hook.Requirements, compContext, None,
                                                 expectedResultType=(type(None), RequirementsResult))
            else:
                logger.info("Patch of component %s will be skipped", componentName)

            c = _createComponentModel(componentName, filePath, compStageDir,
                                     discoveryResult, requirementsResult)
            result.append(c)
        except:
            logger.error("Could not execute discovery hook in file: %s", filePath)
            raise

    return result

def _validateComponentsDependencies(ctx):
    ''' Validates components metadata interdependencies, e.g. cyclic dependencies

    @return: True if the validation succeeded, False otherwise
    @rtype: bool
    '''
    try:
        ctx.getAllComponents(sort=True)
        return True
    except ValueError:
        logger.exception("Component inter-dependencies validation failed")
        return False

def _isSuccessful(components):
    '''Test if all components are ready to be patched.

    @param components: All components which need to be patched
    @type components: list of vmware.patching.data.model.Component

    @return: False if any component report an error mismatch, otherwise True
    @rtype: bool
    '''
    for c in components:
        # Iterate over component requirements mismatches
        for mismatch in c.requirementsResult.mismatches:
            if mismatch.severity == Mismatch.ERROR:
                return False

    return True

def _addUniqueQuestions(questions, new_questions):
    ''' This adds only unique questions and do not add already existing once.
    if it detects that the questions have same id but something is differnet
    it errors out

    @param questions: List of questions to add to unique questions
    @type questions: List of patch_spec.Questions

    @param new_questions: List of questions to try to add
    @type new_questions: List of patch_spec.Questions
    '''
    for question in new_questions:
        # Because we check Ids below we are sure there could be only one question
        # with that ID
        existing_question = next((x for x in questions \
                                    if x.userDataId == question.userDataId), None)
        if not existing_question:
            questions.append(question)
        else:
            if question != existing_question:
                raise ValueError("Cannot have questions with same "
                                    "Id but different text!")

def _validateReplicationConfig(replicationConfigByComponents):
    """
    Validates if there are two components defining a mapping either for the same file on the source
    to different target locations or different source files to the same target location.
    @param replicationConfigByComponents: Holds the replication configuration for each component
    @type replicationConfigByComponents: dict
    @raise ValueError: If there are two components defining a mapping either for the same file on
    the source to different target locations or different source files to the same target location.
    """
    allSourceToTargetFiles = {}
    allTargetToSourceFiles = {}

    for component, replicationConfig in replicationConfigByComponents.items():
        for sourcePath, targetPath in replicationConfig.items():
            if sourcePath not in allSourceToTargetFiles:
                # No other component has defined the file for replication yet.
                allSourceToTargetFiles[sourcePath] = (targetPath, component)
            else:
                alreadyDefinedTargetPath, alreadyDefinedByComponent = allSourceToTargetFiles[sourcePath]
                if alreadyDefinedTargetPath == targetPath:
                    logging.warning("Component %s defines the same replication mapping[%s]:[%s] as component %s",
                                 component,
                                 sourcePath,
                                 targetPath,
                                 alreadyDefinedByComponent)
                    continue
                else:
                    messageTemplate = "Component {} defines mapping [{}]:[{}] for the same file on the source " \
                                      "to different target location than component {} [{}]:[{}]!"
                    message = messageTemplate.format(component,
                                                     sourcePath,
                                                     targetPath,
                                                     alreadyDefinedByComponent,
                                                     sourcePath,
                                                     alreadyDefinedTargetPath)
                    logging.error(message)
                    raise ValueError(message)

            # If the target path is None, we shouldn't validate if more than one sources point to it.
            if targetPath is None or targetPath == "":
                continue

            if targetPath not in allTargetToSourceFiles:
                # No other component has defined replication for a file to the target path yet.
                allTargetToSourceFiles[targetPath] = (sourcePath, component)
            else:
                alreadyDefinedSourcePath, alreadyDefinedByComponent = allTargetToSourceFiles[targetPath]
                if not alreadyDefinedSourcePath == sourcePath:
                    messageTemplate = "Component {} defines mapping [{}]:[{}] for different files on the source " \
                                      "to the same target location as component {} [{}]:[{}]!"
                    message = messageTemplate.format(component,
                                                     sourcePath,
                                                     targetPath,
                                                     alreadyDefinedByComponent,
                                                     alreadyDefinedSourcePath,
                                                     targetPath)
                    logging.error(message)
                    raise ValueError(message)

def _reportResults(components, mismatches, outputFile, stageDir, upgradeType):
    '''Report discovery results into the file.

    @param components: Components which have been discovered.
    @type components: List of vmware.patching.data.model.Component

    @param mismatches: Local mismatches
    @type mismatches: list

    @param outputFile: Output file
    @type outputFile: str
    '''
    # TODO iradev: Consider using real software-update specs rather than
    # serializing to dictionary. This would handle eventual property mismatches
    if not outputFile:
        logger.warning("The output file is not provided. Skip writing results")
        return

    requiredDiskSpace = {}
    # deployment specific rpm ignore list will be added in software-packages
    # integration point
    rpmIgnoreList = []
    questions = []
    rebootRequired = False
    timeToInstall = 0
    # Do not merge patch summary changes from components for now. Leave this to
    # be set on a global level.
    patchSummary = None

    # Holds the entire replication config
    replicationConfig = {}
    # Holds the replication config for all the discovered components.
    replicationConfigByComponents = {}

    for c in components:
        # We need to replicate component's files even if it isn't patchable.
        replicationConfig.update(c.discoveryResult.replicationConfig)
        replicationConfigByComponents[c.name] = c.discoveryResult.replicationConfig
        if not c.discoveryResult.patchable:
            continue

        discoveryResult = c.discoveryResult
        requirementsSpec = c.requirementsResult
        requirements = requirementsSpec.requirements

        # Aggregate required disk space
        for partition, diskRequirement in requirements.requiredDiskSpace.items():
            partitionRequirements = requiredDiskSpace.get(partition, 0.0)
            partitionRequirements += float(diskRequirement)
            requiredDiskSpace[partition] = partitionRequirements

        # Aggregate rpm ignore list
        rpmIgnoreList.extend(discoveryResult.ignoreRpms)

        # Aggregate time to install
        timeToInstall += requirementsSpec.patchInfo.timeToInstall

        # Aggregate questions
        _addUniqueQuestions(questions, requirements.questions)

        # Aggregate reboot required
        rebootRequired = rebootRequired or requirements.rebootRequired

        # Aggregate mismatches
        mismatches.extend(requirementsSpec.mismatches)

    result = {
        'requirements': {
            'requiredDiskSpace': requiredDiskSpace,
            'rpmIgnoreList': rpmIgnoreList,
            'questions': questions,
            'rebootRequired': rebootRequired
        },
        'patchInfo': {
            'timeToInstall': timeToInstall,
            'patchSummary': patchSummary
        },
        'mismatches': mismatches
    }

    if upgradeType == PatchContext.NONDISRUPTIVE_UPGRADE:
        _validateReplicationConfig(replicationConfigByComponents)
        result['replicationConfig'] = replicationConfig

    prettyJsonDump(outputFile, result)

    prechecksFileAbsPath = os.path.join(stageDir, PRECHECKS_FILE)
    shutil.copy(outputFile, prechecksFileAbsPath)



def _validateDeprecatedRpmRemoval(ctx, fmwMismatches):
    '''
    Validates that the removal of deprecated rpm will be successful. If there
    are problems they are appended to fmwMismatches
    '''
    # Gather rpms for removal remove duplicates
    allRpmList = set()
    for component in ctx.getAllComponents():
        allRpmList = allRpmList.union(component.discoveryResult.deprecatedProducts)

    allRpmList = list(allRpmList)  # list is needed for reporting
    result = validateRpmsRemoval(allRpmList)

    if not result:
        mismatch = Mismatch(_(CANNOT_REMOVE_RPM_WARNING_TEXT, ' '.join(allRpmList)), \
                           description=CANNOT_REMOVE_RPM_WARNING_DESCRIPTION, \
                           resolution=CANNOT_REMOVE_RPM_ERROR_RESOLUTION, \
                           severity=Mismatch.WARNING)
        fmwMismatches.append(mismatch)

def _validateStorageForRpmInstallation(workUpdatesDir, fmwMismatches):
    '''
    Validates that there is enough storage for rpm installation. If there are
    problems they are appended to fmwMismatches
    '''
    logger.info('Check if there is enough storage for RPM installation')

    # Update APIs give the b2b different stageDirs for prechecks and the actual
    # pathching. This file has information which is the directory given to the
    # patching but it is a few levels above the stage dir given to b2b so took
    # its path from applmgmt code
    workDir = os.path.join(workUpdatesDir, os.pardir, os.pardir, os.pardir)
    applgmtStageDir = os.path.join(workDir, 'stage')
    stagePathFile = os.path.join(applgmtStageDir, 'stageDir.json')
    logger.info('RPM stage dir is %s', stagePathFile)

    rpmStageDir = None
    if os.path.exists(stagePathFile):
        with open(stagePathFile) as fp:
            rpmStageDir = json.load(fp)['StageDir']
    else:
        # if the file is missing this means that there are no staged rpms
        return

    # The rpm directory might has other files too so we have to prune them
    rpmList = [(os.path.join(rpmStageDir, f)) for f in os.listdir(rpmStageDir) \
        if os.path.isfile(os.path.join(rpmStageDir, f)) and f.endswith('.rpm')]

    mismatches = getInsufficientStorageForRpmInstallation(rpmList)
    if mismatches:
        for k, v in mismatches.items():
            mismatch = Mismatch(_(NOT_ENOUGH_STORAGE_TO_INSTALL_RPM_ERROR_TEXT, k), \
                               description=NOT_ENOUGH_STORAGE_TO_INSTALL_RPM_ERROR_DESCRIPTION, \
                               resolution=_(NOT_ENOUGH_STORAGE_TO_INSTALL_RPM_ERROR_RESOLUTION, [k, str(ceil(v))]))
            fmwMismatches.append(mismatch)

def discover(stageDir,
             outputFile,
             components,
             locale,
             skipPrechecksIds,
             upgradeType=PatchContext.DISRUPTIVE_UPGRADE):
    '''The entry point of discovery CPO phase.

    The discovery phase discovers all components which need to take part in the
    patching process, validates their metadata, validates their inter-component
    dependencies and persist their result for later patch phases. If a component
    wants to take part in the patching workflow, then its requirements will be
    collected(call its requirements hook) as part of discovery process.

    @param stageDir: global CPO stage directory. All components stage
      directories will be created as sub-directories of global stage directory.
    @type stageDir: str

    @param outputFile: File where the output of the command needs to be written
      to
    @type outputFile: str

    @param components: The components which the customer want to patch, the
      other will be filtered out. If None is passed, then all components will be
      returned. The name of the component is equal to the directory where
      its patching logic reside.
    @param components: array of strings

    @param locale: Current locale. All user-facing messages should be
      translated to that locale.
    @type locale: str

    @param skipPrechecksIds: List of ids of prechecks to be skipped.
    @type skipPrechecksIds: list(str)

    @param upgradeType: The type of the upgrade being performed. The allowed values
      are either PatchContext.DISRUPTIVE_UPGRADE or PatchContext.NONDISRUPTIVE_UPGRADE.

    @return: True if the discovery phase succeed and False otherwise
    @rtype: bool

    @raise Exception: if discovery phase raise unhandled exception
    '''
    fmwMismatches = []
    discoveredComponents = []
    ctxFile = os.path.join(stageDir, PatchPhaseContext.CONTEXT_FILE)
    # This CodeBlock will be executed when the Discovery Hook is called during
    # Resume Patch Flow.
    if (os.path.exists(ctxFile) and
            PatchPhaseContext.load(stageDir).isUpgradeStarted()):
        logger.info("This is a Resume Flow, skipping running Discovery hook again")
        prechecksFileAbsPath = os.path.join(stageDir, PRECHECKS_FILE)
        if not os.path.exists(prechecksFileAbsPath):
            logger.error("PreChecks File: %s doesnt exists", prechecksFileAbsPath)
            mismatch = Mismatch(text=INTERNAL_ERROR_TEXT,
                                description=INTERNAL_ERROR_TEXT,
                                resolution=INTERNAL_ERROR_RESOLUTION)
            fmwMismatches.append(mismatch)
            _reportResults(discoveredComponents, fmwMismatches, outputFile, stageDir, upgradeType)
        else:
            shutil.copy(prechecksFileAbsPath, outputFile)

        return

    logger.info("Do discovery... Components: %s, locale: %s", components, locale)
    configureLocalization(locale)
    _cleanupRootStageDir(stageDir)
    try:
        result = False
        # Check if vCHA is configured on the given vCenter Server or not.
        if _isVCHAConfigured():
            mismatch = Mismatch(text=CANNOT_PATCH_ON_VC_CONFIGURED_WITH_VCHA_ERROR_TEXT,
                                resolution=CANNOT_PATCH_ON_VC_CONFIGURED_WITH_VCHA_ERROR_RESOLUTION)
            fmwMismatches.append(mismatch)
            return result

        scriptsRootDir = getPayloadFilePath('components-script')
        discoveredComponents = _discoverComponents(scriptsRootDir, stageDir, components,
                                              locale, skipPrechecksIds, upgradeType)
        patchableComponents = [c for c in discoveredComponents if c.discoveryResult.patchable]
        # Log which components have been found
        logger.info("Discovery completed. Result: %s\n",
                    json.dumps(discoveredComponents,
                               default=ObjectJsonConverter(False).encode,
                               indent=4,
                               sort_keys=True))

        ctx = PatchPhaseContext(stageDir, patchableComponents, locale, upgradeType=upgradeType)
        if not _validateComponentsDependencies(ctx):
            mismatch = Mismatch(text=INTERNAL_ERROR_TEXT,
                            description=INTERNAL_ERROR_TEXT,
                            resolution=INTERNAL_ERROR_RESOLUTION)
            fmwMismatches.append(mismatch)
        else:
            result = _isSuccessful(patchableComponents)

        if result:
            ctx.persist(stageDir)


        if (upgradeType == PatchContext.DISRUPTIVE_UPGRADE):
            _validateDeprecatedRpmRemoval(ctx, fmwMismatches)

            # ! TODO The right place for this check is in the update APIs where the RPMs are installed
            # ! however we want to catch the next relase so this is done here.
            # ! Revisit it after the release
            # Check if there is enough storage for the rpm installation
            _validateStorageForRpmInstallation(stageDir, fmwMismatches)

        return result
    except ComponentError as wrapperError:
        logger.exception("Discovery hook got ComponentWrapperError.")
        if isinstance(wrapperError.baseError, UserError):
            mismatch = Mismatch(text=wrapperError.baseError.cause,
                        description=wrapperError.baseError.cause,
                        resolution=wrapperError.baseError.resolution)
        else:
            mismatch = Mismatch(text=INTERNAL_ERROR_TEXT,
                        description=INTERNAL_ERROR_TEXT,
                        resolution=INTERNAL_ERROR_RESOLUTION)
        fmwMismatches.append(mismatch)
        return False
    except UserError as userError:
        logger.exception("Discovery hook got UserError.")
        mismatch = Mismatch(text=userError.cause,
                        description=userError.cause,
                        resolution=userError.resolution)
        fmwMismatches.append(mismatch)
        return False
    except Exception:  # pylint: disable=W0703
        logger.exception("Discovery hook got Exception.")
        mismatch = Mismatch(text=INTERNAL_ERROR_TEXT,
                        description=INTERNAL_ERROR_TEXT,
                        resolution=INTERNAL_ERROR_RESOLUTION)
        fmwMismatches.append(mismatch)
        return False
    finally:
        _reportResults(discoveredComponents, fmwMismatches, outputFile, stageDir, upgradeType)
