manifest_xml.py

# Copyright (C) 2008 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import itertools
import os
import platform
import re
import sys
import urllib.parse
import xml.dom.minidom

from error import ManifestInvalidPathError
from error import ManifestInvalidRevisionError
from error import ManifestParseError
from git_config import GitConfig
from git_refs import HEAD
from git_refs import R_HEADS
from git_superproject import Superproject
import platform_utils
from project import Annotation
from project import ManifestProject
from project import Project
from project import RemoteSpec
from project import RepoProject
from wrapper import Wrapper


MANIFEST_FILE_NAME = "manifest.xml"
LOCAL_MANIFEST_NAME = "local_manifest.xml"
LOCAL_MANIFESTS_DIR_NAME = "local_manifests"
SUBMANIFEST_DIR = "submanifests"
# Limit submanifests to an arbitrary depth for loop detection.
MAX_SUBMANIFEST_DEPTH = 8
# Add all projects from sub manifest into a group.
SUBMANIFEST_GROUP_PREFIX = "submanifest:"

# Add all projects from local manifest into a group.
LOCAL_MANIFEST_GROUP_PREFIX = "local:"

# ContactInfo has the self-registered bug url, supplied by the manifest authors.
ContactInfo = collections.namedtuple("ContactInfo", "bugurl")

# urljoin gets confused if the scheme is not known.
urllib.parse.uses_relative.extend(
    ["ssh", "git", "persistent-https", "sso", "rpc"]
)
urllib.parse.uses_netloc.extend(
    ["ssh", "git", "persistent-https", "sso", "rpc"]
)


def XmlBool(node, attr, default=None):
    """Determine boolean value of |node|'s |attr|.

    Invalid values will issue a non-fatal warning.

    Args:
        node: XML node whose attributes we access.
        attr: The attribute to access.
        default: If the attribute is not set (value is empty), then use this.

    Returns:
        True if the attribute is a valid string representing true.
        False if the attribute is a valid string representing false.
        |default| otherwise.
    """
    value = node.getAttribute(attr)
    s = value.lower()
    if s == "":
        return default
    elif s in {"yes", "true", "1"}:
        return True
    elif s in {"no", "false", "0"}:
        return False
    else:
        print(
            'warning: manifest: %s="%s": ignoring invalid XML boolean'
            % (attr, value),
            file=sys.stderr,
        )
        return default


def XmlInt(node, attr, default=None):
    """Determine integer value of |node|'s |attr|.

    Args:
        node: XML node whose attributes we access.
        attr: The attribute to access.
        default: If the attribute is not set (value is empty), then use this.

    Returns:
        The number if the attribute is a valid number.

    Raises:
        ManifestParseError: The number is invalid.
    """
    value = node.getAttribute(attr)
    if not value:
        return default

    try:
        return int(value)
    except ValueError:
        raise ManifestParseError(f'manifest: invalid {attr}="{value}" integer')


def normalize_url(url: str) -> str:
    """Mutate input 'url' into normalized form:

    * remove trailing slashes
    * convert SCP-like syntax to SSH URL

    Args:
        url: URL to modify

    Returns:
        The normalized URL.
    """

    url = url.rstrip("/")
    parsed_url = urllib.parse.urlparse(url)

    # This matches patterns like "git@github.com:foo".
    scp_like_url_re = r"^[^/:]+@[^/:]+:[^/]+"

    # If our URL is missing a schema and matches git's
    # SCP-like syntax we should convert it to a proper
    # SSH URL instead to make urljoin() happier.
    #
    # See: https://git-scm.com/docs/git-clone#URLS
    if not parsed_url.scheme and re.match(scp_like_url_re, url):
        return "ssh://" + url.replace(":", "/", 1)

    return url


class _Default:
    """Project defaults within the manifest."""

    revisionExpr = None
    destBranchExpr = None
    upstreamExpr = None
    remote = None
    sync_j = None
    sync_c = False
    sync_s = False
    sync_tags = True

    def __eq__(self, other):
        if not isinstance(other, _Default):
            return False
        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        if not isinstance(other, _Default):
            return True
        return self.__dict__ != other.__dict__


class _XmlRemote:
    def __init__(
        self,
        name,
        alias=None,
        fetch=None,
        pushUrl=None,
        manifestUrl=None,
        review=None,
        revision=None,
    ):
        self.name = name
        self.fetchUrl = fetch
        self.pushUrl = pushUrl
        self.manifestUrl = manifestUrl
        self.remoteAlias = alias
        self.reviewUrl = review
        self.revision = revision
        self.resolvedFetchUrl = self._resolveFetchUrl()
        self.annotations = []

    def __eq__(self, other):
        if not isinstance(other, _XmlRemote):
            return False
        return (
            sorted(self.annotations) == sorted(other.annotations)
            and self.name == other.name
            and self.fetchUrl == other.fetchUrl
            and self.pushUrl == other.pushUrl
            and self.remoteAlias == other.remoteAlias
            and self.reviewUrl == other.reviewUrl
            and self.revision == other.revision
        )

    def __ne__(self, other):
        return not self.__eq__(other)

    def _resolveFetchUrl(self):
        if self.fetchUrl is None:
            return ""

        fetch_url = normalize_url(self.fetchUrl)
        manifest_url = normalize_url(self.manifestUrl)

        # urljoin doesn't like URLs with no scheme in the base URL
        # such as file paths.  We handle this by prefixing it with
        # an obscure protocol, gopher, and replacing it with the
        # original after urljoin
        if manifest_url.find(":") != manifest_url.find("/") - 1:
            fetch_url = urllib.parse.urljoin(
                "gopher://" + manifest_url, fetch_url
            )
            fetch_url = re.sub(r"^gopher://", "", fetch_url)
        else:
            fetch_url = urllib.parse.urljoin(manifest_url, fetch_url)
        return fetch_url

    def ToRemoteSpec(self, projectName):
        fetchUrl = self.resolvedFetchUrl.rstrip("/")
        url = fetchUrl + "/" + projectName
        remoteName = self.name
        if self.remoteAlias:
            remoteName = self.remoteAlias
        return RemoteSpec(
            remoteName,
            url=url,
            pushUrl=self.pushUrl,
            review=self.reviewUrl,
            orig_name=self.name,
            fetchUrl=self.fetchUrl,
        )

    def AddAnnotation(self, name, value, keep):
        self.annotations.append(Annotation(name, value, keep))


class _XmlSubmanifest:
    """Manage the <submanifest> element specified in the manifest.

    Attributes:
      name: a string, the name for this submanifest.
      remote: a string, the remote.name for this submanifest.
      project: a string, the name of the manifest project.
      revision: a string, the commitish.
      manifestName: a string, the submanifest file name.
      groups: a list of strings, the groups to add to all projects in the
          submanifest.
      default_groups: a list of strings, the default groups to sync.
      path: a string, the relative path for the submanifest checkout.
      parent: an XmlManifest, the parent manifest.
      annotations: (derived) a list of annotations.
      present: (derived) a boolean, whether the sub manifest file is present.
    """

    def __init__(
        self,
        name,
        remote=None,
        project=None,
        revision=None,
        manifestName=None,
        groups=None,
        default_groups=None,
        path=None,
        parent=None,
    ):
        self.name = name
        self.remote = remote
        self.project = project
        self.revision = revision
        self.manifestName = manifestName
        self.groups = groups
        self.default_groups = default_groups
        self.path = path
        self.parent = parent
        self.annotations = []
        outer_client = parent._outer_client or parent
        if self.remote and not self.project:
            raise ManifestParseError(
                f"Submanifest {name}: must specify project when remote is "
                "given."
            )
        # Construct the absolute path to the manifest file using the parent's
        # method, so that we can correctly create our repo_client.
        manifestFile = parent.SubmanifestInfoDir(
            os.path.join(parent.path_prefix, self.relpath),
            os.path.join("manifests", manifestName or "default.xml"),
        )
        linkFile = parent.SubmanifestInfoDir(
            os.path.join(parent.path_prefix, self.relpath), MANIFEST_FILE_NAME
        )
        self.repo_client = RepoClient(
            parent.repodir,
            linkFile,
            parent_groups=",".join(groups) or "",
            submanifest_path=os.path.join(parent.path_prefix, self.relpath),
            outer_client=outer_client,
            default_groups=default_groups,
        )

        self.present = os.path.exists(manifestFile)

    def __eq__(self, other):
        if not isinstance(other, _XmlSubmanifest):
            return False
        return (
            self.name == other.name
            and self.remote == other.remote
            and self.project == other.project
            and self.revision == other.revision
            and self.manifestName == other.manifestName
            and self.groups == other.groups
            and self.default_groups == other.default_groups
            and self.path == other.path
            and sorted(self.annotations) == sorted(other.annotations)
        )

    def __ne__(self, other):
        return not self.__eq__(other)

    def ToSubmanifestSpec(self):
        """Return a SubmanifestSpec object, populating attributes"""
        mp = self.parent.manifestProject
        remote = self.parent.remotes[
            self.remote or self.parent.default.remote.name
        ]
        # If a project was given, generate the url from the remote and project.
        # If not, use this manifestProject's url.
        if self.project:
            manifestUrl = remote.ToRemoteSpec(self.project).url
        else:
            manifestUrl = mp.GetRemote().url
        manifestName = self.manifestName or "default.xml"
        revision = self.revision or self.name
        path = self.path or revision.split("/")[-1]
        groups = self.groups or []

        return SubmanifestSpec(
            self.name, manifestUrl, manifestName, revision, path, groups
        )

    @property
    def relpath(self):
        """The path of this submanifest relative to the parent manifest."""
        revision = self.revision or self.name
        return self.path or revision.split("/")[-1]

    def GetGroupsStr(self):
        """Returns the `groups` given for this submanifest."""
        if self.groups:
            return ",".join(self.groups)
        return ""

    def GetDefaultGroupsStr(self):
        """Returns the `default-groups` given for this submanifest."""
        return ",".join(self.default_groups or [])

    def AddAnnotation(self, name, value, keep):
        """Add annotations to the submanifest."""
        self.annotations.append(Annotation(name, value, keep))


class SubmanifestSpec:
    """The submanifest element, with all fields expanded."""

    def __init__(self, name, manifestUrl, manifestName, revision, path, groups):
        self.name = name
        self.manifestUrl = manifestUrl
        self.manifestName = manifestName
        self.revision = revision
        self.path = path
        self.groups = groups or []


class XmlManifest:
    """manages the repo configuration file"""

    def __init__(
        self,
        repodir,
        manifest_file,
        local_manifests=None,
        outer_client=None,
        parent_groups="",
        submanifest_path="",
        default_groups=None,
    ):
        """Initialize.

        Args:
            repodir: Path to the .repo/ dir for holding all internal checkout
                state. It must be in the top directory of the repo client
                checkout.
            manifest_file: Full path to the manifest file to parse.  This will
                usually be |repodir|/|MANIFEST_FILE_NAME|.
            local_manifests: Full path to the directory of local override
                manifests. This will usually be
                |repodir|/|LOCAL_MANIFESTS_DIR_NAME|.
            outer_client: RepoClient of the outer manifest.
            parent_groups: a string, the groups to apply to this projects.
            submanifest_path: The submanifest root relative to the repo root.
            default_groups: a string, the default manifest groups to use.
        """
        # TODO(vapier): Move this out of this class.
        self.globalConfig = GitConfig.ForUser()

        self.repodir = os.path.abspath(repodir)
        self._CheckLocalPath(submanifest_path)
        self.topdir = os.path.dirname(self.repodir)
        if submanifest_path:
            # This avoids a trailing os.path.sep when submanifest_path is empty.
            self.topdir = os.path.join(self.topdir, submanifest_path)
        if manifest_file != os.path.abspath(manifest_file):
            raise ManifestParseError("manifest_file must be abspath")
        self.manifestFile = manifest_file
        if not outer_client or outer_client == self:
            # manifestFileOverrides only exists in the outer_client's manifest,
            # since that is the only instance left when Unload() is called on
            # the outer manifest.
            self.manifestFileOverrides = {}
        self.local_manifests = local_manifests
        self._load_local_manifests = True
        self.parent_groups = parent_groups
        self.default_groups = default_groups

        if submanifest_path and not outer_client:
            # If passing a submanifest_path, there must be an outer_client.
            raise ManifestParseError(f"Bad call to {self.__class__.__name__}")

        # If self._outer_client is None, this is not a checkout that supports
        # multi-tree.
        self._outer_client = outer_client or self

        self.repoProject = RepoProject(
            self,
            "repo",
            gitdir=os.path.join(repodir, "repo/.git"),
            worktree=os.path.join(repodir, "repo"),
        )

        mp = self.SubmanifestProject(self.path_prefix)
        self.manifestProject = mp

        # This is a bit hacky, but we're in a chicken & egg situation: all the
        # normal repo settings live in the manifestProject which we just setup
        # above, so we couldn't easily query before that.  We assume Project()
        # init doesn't care if this changes afterwards.
        if os.path.exists(mp.gitdir) and mp.use_worktree:
            mp.use_git_worktrees = True

        self.Unload()

    def Override(self, name, load_local_manifests=True):
        """Use a different manifest, just for the current instantiation."""
        path = None

        # Look for a manifest by path in the filesystem (including the cwd).
        if not load_local_manifests:
            local_path = os.path.abspath(name)
            if os.path.isfile(local_path):
                path = local_path

        # Look for manifests by name from the manifests repo.
        if path is None:
            path = os.path.join(self.manifestProject.worktree, name)
            if not os.path.isfile(path):
                raise ManifestParseError("manifest %s not found" % name)

        self._load_local_manifests = load_local_manifests
        self._outer_client.manifestFileOverrides[self.path_prefix] = path
        self.Unload()
        self._Load()

    def Link(self, name):
        """Update the repo metadata to use a different manifest."""
        self.Override(name)

        # Old versions of repo would generate symlinks we need to clean up.
        platform_utils.remove(self.manifestFile, missing_ok=True)
        # This file is interpreted as if it existed inside the manifest repo.
        # That allows us to use <include> with the relative file name.
        with open(self.manifestFile, "w") as fp:
            fp.write(
                """<?xml version="1.0" encoding="UTF-8"?>
<!--
DO NOT EDIT THIS FILE!  It is generated by repo and changes will be discarded.
If you want to use a different manifest, use `repo init -m <file>` instead.

If you want to customize your checkout by overriding manifest settings, use
the local_manifests/ directory instead.

For more information on repo manifests, check out:
https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md
-->
<manifest>
  <include name="%s" />
</manifest>
"""
                % (name,)
            )

    def _RemoteToXml(self, r, doc, root):
        e = doc.createElement("remote")
        root.appendChild(e)
        e.setAttribute("name", r.name)
        e.setAttribute("fetch", r.fetchUrl)
        if r.pushUrl is not None:
            e.setAttribute("pushurl", r.pushUrl)
        if r.remoteAlias is not None:
            e.setAttribute("alias", r.remoteAlias)
        if r.reviewUrl is not None:
            e.setAttribute("review", r.reviewUrl)
        if r.revision is not None:
            e.setAttribute("revision", r.revision)

        for a in r.annotations:
            if a.keep == "true":
                ae = doc.createElement("annotation")
                ae.setAttribute("name", a.name)
                ae.setAttribute("value", a.value)
                e.appendChild(ae)

    def _SubmanifestToXml(self, r, doc, root):
        """Generate XML <submanifest/> node."""
        e = doc.createElement("submanifest")
        root.appendChild(e)
        e.setAttribute("name", r.name)
        if r.remote is not None:
            e.setAttribute("remote", r.remote)
        if r.project is not None:
            e.setAttribute("project", r.project)
        if r.manifestName is not None:
            e.setAttribute("manifest-name", r.manifestName)
        if r.revision is not None:
            e.setAttribute("revision", r.revision)
        if r.path is not None:
            e.setAttribute("path", r.path)
        if r.groups:
            e.setAttribute("groups", r.GetGroupsStr())
        if r.default_groups:
            e.setAttribute("default-groups", r.GetDefaultGroupsStr())

        for a in r.annotations:
            if a.keep == "true":
                ae = doc.createElement("annotation")
                ae.setAttribute("name", a.name)
                ae.setAttribute("value", a.value)
                e.appendChild(ae)

    def _ParseList(self, field):
        """Parse fields that contain flattened lists.

        These are whitespace & comma separated.  Empty elements will be
        discarded.
        """
        return [x for x in re.split(r"[,\s]+", field) if x]

    def ToXml(
        self,
        peg_rev=False,
        peg_rev_upstream=True,
        peg_rev_dest_branch=True,
        groups=None,
        omit_local=False,
    ):
        """Return the current manifest XML."""
        mp = self.manifestProject

        if groups is None:
            groups = mp.manifest_groups
        if groups:
            groups = self._ParseList(groups)

        doc = xml.dom.minidom.Document()
        root = doc.createElement("manifest")
        if self.is_submanifest:
            root.setAttribute("path", self.path_prefix)
        doc.appendChild(root)

        # Save out the notice.  There's a little bit of work here to give it the
        # right whitespace, which assumes that the notice is automatically
        # indented by 4 by minidom.
        if self.notice:
            notice_element = root.appendChild(doc.createElement("notice"))
            notice_lines = self.notice.splitlines()
            indented_notice = (
                "\n".join(" " * 4 + line for line in notice_lines)
            )[4:]
            notice_element.appendChild(doc.createTextNode(indented_notice))

        d = self.default

        for r in sorted(self.remotes):
            self._RemoteToXml(self.remotes[r], doc, root)
        if self.remotes:
            root.appendChild(doc.createTextNode(""))

        have_default = False
        e = doc.createElement("default")
        if d.remote:
            have_default = True
            e.setAttribute("remote", d.remote.name)
        if d.revisionExpr:
            have_default = True
            e.setAttribute("revision", d.revisionExpr)
        if d.destBranchExpr:
            have_default = True
            e.setAttribute("dest-branch", d.destBranchExpr)
        if d.upstreamExpr:
            have_default = True
            e.setAttribute("upstream", d.upstreamExpr)
        if d.sync_j is not None:
            have_default = True
            e.setAttribute("sync-j", "%d" % d.sync_j)
        if d.sync_c:
            have_default = True
            e.setAttribute("sync-c", "true")
        if d.sync_s:
            have_default = True
            e.setAttribute("sync-s", "true")
        if not d.sync_tags:
            have_default = True
            e.setAttribute("sync-tags", "false")
        if have_default:
            root.appendChild(e)
            root.appendChild(doc.createTextNode(""))

        if self._manifest_server:
            e = doc.createElement("manifest-server")
            e.setAttribute("url", self._manifest_server)
            root.appendChild(e)
            root.appendChild(doc.createTextNode(""))

        for r in sorted(self.submanifests):
            self._SubmanifestToXml(self.submanifests[r], doc, root)
        if self.submanifests:
            root.appendChild(doc.createTextNode(""))

        def output_projects(parent, parent_node, projects):
            for project_name in projects:
                for project in self._projects[project_name]:
                    output_project(parent, parent_node, project)

        def output_project(parent, parent_node, p):
            if not p.MatchesGroups(groups):
                return

            if omit_local and self.IsFromLocalManifest(p):
                return

            name = p.name
            relpath = p.relpath
            if parent:
                name = self._UnjoinName(parent.name, name)
                relpath = self._UnjoinRelpath(parent.relpath, relpath)

            e = doc.createElement("project")
            parent_node.appendChild(e)
            e.setAttribute("name", name)
            if relpath != name:
                e.setAttribute("path", relpath)
            remoteName = None
            if d.remote:
                remoteName = d.remote.name
            if not d.remote or p.remote.orig_name != remoteName:
                remoteName = p.remote.orig_name
                e.setAttribute("remote", remoteName)
            if peg_rev:
                if self.IsMirror:
                    value = p.bare_git.rev_parse(p.revisionExpr + "^0")
                else:
                    value = p.work_git.rev_parse(HEAD + "^0")
                e.setAttribute("revision", value)
                if peg_rev_upstream:
                    if p.upstream:
                        e.setAttribute("upstream", p.upstream)
                    elif value != p.revisionExpr:
                        # Only save the origin if the origin is not a sha1, and
                        # the default isn't our value
                        e.setAttribute("upstream", p.revisionExpr)

                if peg_rev_dest_branch:
                    if p.dest_branch:
                        e.setAttribute("dest-branch", p.dest_branch)
                    elif value != p.revisionExpr:
                        e.setAttribute("dest-branch", p.revisionExpr)

            else:
                revision = (
                    self.remotes[p.remote.orig_name].revision or d.revisionExpr
                )
                if not revision or revision != p.revisionExpr:
                    e.setAttribute("revision", p.revisionExpr)
                elif p.revisionId:
                    e.setAttribute("revision", p.revisionId)
                if p.upstream and (
                    p.upstream != p.revisionExpr or p.upstream != d.upstreamExpr
                ):
                    e.setAttribute("upstream", p.upstream)

            if p.dest_branch and p.dest_branch != d.destBranchExpr:
                e.setAttribute("dest-branch", p.dest_branch)

            for c in p.copyfiles:
                ce = doc.createElement("copyfile")
                ce.setAttribute("src", c.src)
                ce.setAttribute("dest", c.dest)
                e.appendChild(ce)

            for lf in p.linkfiles:
                le = doc.createElement("linkfile")
                le.setAttribute("src", lf.src)
                le.setAttribute("dest", lf.dest)
                e.appendChild(le)

            default_groups = ["all", "name:%s" % p.name, "path:%s" % p.relpath]
            egroups = [g for g in p.groups if g not in default_groups]
            if egroups:
                e.setAttribute("groups", ",".join(egroups))

            for a in p.annotations:
                if a.keep == "true":
                    ae = doc.createElement("annotation")
                    ae.setAttribute("name", a.name)
                    ae.setAttribute("value", a.value)
                    e.appendChild(ae)

            if p.sync_c:
                e.setAttribute("sync-c", "true")

            if p.sync_s:
                e.setAttribute("sync-s", "true")

            if not p.sync_tags:
                e.setAttribute("sync-tags", "false")

            if p.clone_depth:
                e.setAttribute("clone-depth", str(p.clone_depth))

            self._output_manifest_project_extras(p, e)

            if p.subprojects:
                subprojects = {subp.name for subp in p.subprojects}
                output_projects(p, e, list(sorted(subprojects)))

        projects = {p.name for p in self._paths.values() if not p.parent}
        output_projects(None, root, list(sorted(projects)))

        if self._repo_hooks_project:
            root.appendChild(doc.createTextNode(""))
            e = doc.createElement("repo-hooks")
            e.setAttribute("in-project", self._repo_hooks_project.name)
            e.setAttribute(
                "enabled-list",
                " ".join(self._repo_hooks_project.enabled_repo_hooks),
            )
            root.appendChild(e)

        if self._superproject:
            root.appendChild(doc.createTextNode(""))
            e = doc.createElement("superproject")
            e.setAttribute("name", self._superproject.name)
            remoteName = None
            if d.remote:
                remoteName = d.remote.name
            remote = self._superproject.remote
            if not d.remote or remote.orig_name != remoteName:
                remoteName = remote.orig_name
                e.setAttribute("remote", remoteName)
            revision = remote.revision or d.revisionExpr
            if not revision or revision != self._superproject.revision:
                e.setAttribute("revision", self._superproject.revision)
            root.appendChild(e)

        if self._contactinfo.bugurl != Wrapper().BUG_URL:
            root.appendChild(doc.createTextNode(""))
            e = doc.createElement("contactinfo")
            e.setAttribute("bugurl", self._contactinfo.bugurl)
            root.appendChild(e)

        return doc

    def ToDict(self, **kwargs):
        """Return the current manifest as a dictionary."""
        # Elements that may only appear once.
        SINGLE_ELEMENTS = {
            "notice",
            "default",
            "manifest-server",
            "repo-hooks",
            "superproject",
            "contactinfo",
        }
        # Elements that may be repeated.
        MULTI_ELEMENTS = {
            "remote",
            "remove-project",
            "project",
            "extend-project",
            "include",
            "submanifest",
            # These are children of 'project' nodes.
            "annotation",
            "project",
            "copyfile",
            "linkfile",
        }

        doc = self.ToXml(**kwargs)
        ret = {}

        def append_children(ret, node):
            for child in node.childNodes:
                if child.nodeType == xml.dom.Node.ELEMENT_NODE:
                    attrs = child.attributes
                    element = {
                        attrs.item(i).localName: attrs.item(i).value
                        for i in range(attrs.length)
                    }
                    if child.nodeName in SINGLE_ELEMENTS:
                        ret[child.nodeName] = element
                    elif child.nodeName in MULTI_ELEMENTS:
                        ret.setdefault(child.nodeName, []).append(element)
                    else:
                        raise ManifestParseError(
                            f'Unhandled element "{child.nodeName}"'
                        )

                    append_children(element, child)

        append_children(ret, doc.firstChild)

        return ret

    def Save(self, fd, **kwargs):
        """Write the current manifest out to the given file descriptor."""
        doc = self.ToXml(**kwargs)
        doc.writexml(fd, "", "  ", "\n", "UTF-8")

    def _output_manifest_project_extras(self, p, e):
        """Manifests can modify e if they support extra project attributes."""

    @property
    def is_multimanifest(self):
        """Whether this is a multimanifest checkout.

        This is safe to use as long as the outermost manifest XML has been
        parsed.
        """
        return bool(self._outer_client._submanifests)

    @property
    def is_submanifest(self):
        """Whether this manifest is a submanifest.

        This is safe to use as long as the outermost manifest XML has been
        parsed.
        """
        return self._outer_client and self._outer_client != self

    @property
    def outer_client(self):
        """The instance of the outermost manifest client."""
        self._Load()
        return self._outer_client

    @property
    def all_manifests(self):
        """Generator yielding all (sub)manifests, in depth-first order."""
        self._Load()
        outer = self._outer_client
        yield outer
        yield from outer.all_children

    @property
    def all_children(self):
        """Generator yielding all (present) child submanifests."""
        self._Load()
        for child in self._submanifests.values():
            if child.repo_client:
                yield child.repo_client
                yield from child.repo_client.all_children

    @property
    def path_prefix(self):
        """The path of this submanifest, relative to the outermost manifest."""
        if not self._outer_client or self == self._outer_client:
            return ""
        return os.path.relpath(self.topdir, self._outer_client.topdir)

    @property
    def all_paths(self):
        """All project paths for all (sub)manifests.

        See also `paths`.

        Returns:
            A dictionary of {path: Project()}.  `path` is relative to the outer
            manifest.
        """
        ret = {}
        for tree in self.all_manifests:
            prefix = tree.path_prefix
            ret.update(
                {os.path.join(prefix, k): v for k, v in tree.paths.items()}
            )
        return ret

    @property
    def all_projects(self):
        """All projects for all (sub)manifests.  See `projects`."""
        return list(
            itertools.chain.from_iterable(
                x._paths.values() for x in self.all_manifests
            )
        )

    @property
    def paths(self):
        """Return all paths for this manifest.

        Returns:
            A dictionary of {path: Project()}.  `path` is relative to this
            manifest.
        """
        self._Load()
        return self._paths

    @property
    def projects(self):
        """Return a list of all Projects in this manifest."""
        self._Load()
        return list(self._paths.values())

    @property
    def remotes(self):
        """Return a list of remotes for this manifest."""
        self._Load()
        return self._remotes

    @property
    def default(self):
        """Return default values for this manifest."""
        self._Load()
        return self._default

    @property
    def submanifests(self):
        """All submanifests in this manifest."""
        self._Load()
        return self._submanifests

    @property
    def repo_hooks_project(self):
        self._Load()
        return self._repo_hooks_project

    @property
    def superproject(self):
        self._Load()
        return self._superproject

    @property
    def contactinfo(self):
        self._Load()
        return self._contactinfo

    @property
    def notice(self):
        self._Load()
        return self._notice

    @property
    def manifest_server(self):
        self._Load()
        return self._manifest_server

    @property
    def CloneBundle(self):
        clone_bundle = self.manifestProject.clone_bundle
        if clone_bundle is None:
            return False if self.manifestProject.partial_clone else True
        else:
            return clone_bundle

    @property
    def CloneFilter(self):
        if self.manifestProject.partial_clone:
            return self.manifestProject.clone_filter
        return None

    @property
    def CloneFilterForDepth(self):
        if self.manifestProject.clone_filter_for_depth:
            return self.manifestProject.clone_filter_for_depth
        return None

    @property
    def PartialCloneExclude(self):
        exclude = self.manifest.manifestProject.partial_clone_exclude or ""
        return {x.strip() for x in exclude.split(",")}

    def SetManifestOverride(self, path):
        """Override manifestFile.  The caller must call Unload()"""
        self._outer_client.manifest.manifestFileOverrides[
            self.path_prefix
        ] = path

    @property
    def UseLocalManifests(self):
        return self._load_local_manifests

    def SetUseLocalManifests(self, value):
        self._load_local_manifests = value

    @property
    def HasLocalManifests(self):
        return self._load_local_manifests and self.local_manifests

    def IsFromLocalManifest(self, project):
        """Is the project from a local manifest?"""
        return any(
            x.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for x in project.groups
        )

    @property
    def IsMirror(self):
        return self.manifestProject.mirror

    @property
    def UseGitWorktrees(self):
        return self.manifestProject.use_worktree

    @property
    def IsArchive(self):
        return self.manifestProject.archive

    @property
    def HasSubmodules(self):
        return self.manifestProject.submodules

    @property
    def EnableGitLfs(self):
        return self.manifestProject.git_lfs

    def FindManifestByPath(self, path):
        """Returns the manifest containing path."""
        path = os.path.abspath(path)
        manifest = self._outer_client or self
        old = None
        while manifest._submanifests and manifest != old:
            old = manifest
            for name in manifest._submanifests:
                tree = manifest._submanifests[name]
                if path.startswith(tree.repo_client.manifest.topdir):
                    manifest = tree.repo_client
                    break
        return manifest

    @property
    def subdir(self):
        """Returns the path for per-submanifest objects for this manifest."""
        return self.SubmanifestInfoDir(self.path_prefix)

    def SubmanifestInfoDir(self, submanifest_path, object_path=""):
        """Return the path to submanifest-specific info for a submanifest.

        Return the full path of the directory in which to put per-manifest
        objects.

        Args:
            submanifest_path: a string, the path of the submanifest, relative to
                the outermost topdir.  If empty, then repodir is returned.
            object_path: a string, relative path to append to the submanifest
                info directory path.
        """
        if submanifest_path:
            return os.path.join(
                self.repodir, SUBMANIFEST_DIR, submanifest_path, object_path
            )
        else:
            return os.path.join(self.repodir, object_path)

    def SubmanifestProject(self, submanifest_path):
        """Return a manifestProject for a submanifest."""
        subdir = self.SubmanifestInfoDir(submanifest_path)
        mp = ManifestProject(
            self,
            "manifests",
            gitdir=os.path.join(subdir, "manifests.git"),
            worktree=os.path.join(subdir, "manifests"),
        )
        return mp

    def GetDefaultGroupsStr(self, with_platform=True):
        """Returns the default group string to use.

        Args:
            with_platform: a boolean, whether to include the group for the
                          underlying platform.
        """
        groups = ",".join(self.default_groups or ["default"])
        if with_platform:
            groups += f",platform-{platform.system().lower()}"
        return groups

    def GetGroupsStr(self):
        """Returns the manifest group string that should be synced."""
        return (
            self.manifestProject.manifest_groups or self.GetDefaultGroupsStr()
        )

    def Unload(self):
        """Unload the manifest.

        If the manifest files have been changed since Load() was called, this
        will cause the new/updated manifest to be used.

        """
        self._loaded = False
        self._projects = {}
        self._paths = {}
        self._remotes = {}
        self._default = None
        self._submanifests = {}
        self._repo_hooks_project = None
        self._superproject = None
        self._contactinfo = ContactInfo(Wrapper().BUG_URL)
        self._notice = None
        self.branch = None
        self._manifest_server = None

    def Load(self):
        """Read the manifest into memory."""
        # Do not expose internal arguments.
        self._Load()

    def _Load(self, initial_client=None, submanifest_depth=0):
        if submanifest_depth > MAX_SUBMANIFEST_DEPTH:
            raise ManifestParseError(
                "maximum submanifest depth %d exceeded." % MAX_SUBMANIFEST_DEPTH
            )
        if not self._loaded:
            if self._outer_client and self._outer_client != self:
                # This will load all clients.
                self._outer_client._Load(initial_client=self)

            savedManifestFile = self.manifestFile
            override = self._outer_client.manifestFileOverrides.get(
                self.path_prefix
            )
            if override:
                self.manifestFile = override

            try:
                m = self.manifestProject
                b = m.GetBranch(m.CurrentBranch).merge
                if b is not None and b.startswith(R_HEADS):
                    b = b[len(R_HEADS) :]
                self.branch = b

                parent_groups = self.parent_groups
                if self.path_prefix:
                    parent_groups = (
                        f"{SUBMANIFEST_GROUP_PREFIX}:path:"
                        f"{self.path_prefix},{parent_groups}"
                    )

                # The manifestFile was specified by the user which is why we
                # allow include paths to point anywhere.
                nodes = []
                nodes.append(
                    self._ParseManifestXml(
                        self.manifestFile,
                        self.manifestProject.worktree,
                        parent_groups=parent_groups,
                        restrict_includes=False,
                    )
                )

                if self._load_local_manifests and self.local_manifests:
                    try:
                        for local_file in sorted(
                            platform_utils.listdir(self.local_manifests)
                        ):
                            if local_file.endswith(".xml"):
                                local = os.path.join(
                                    self.local_manifests, local_file
                                )
                                # Since local manifests are entirely managed by
                                # the user, allow them to point anywhere the
                                # user wants.
                                local_group = (
                                    f"{LOCAL_MANIFEST_GROUP_PREFIX}:"
                                    f"{local_file[:-4]}"
                                )
                                nodes.append(
                                    self._ParseManifestXml(
                                        local,
                                        self.subdir,
                                        parent_groups=(
                                            f"{local_group},{parent_groups}"
                                        ),
                                        restrict_includes=False,
                                    )
                                )
                    except OSError:
                        pass

                try:
                    self._ParseManifest(nodes)
                except ManifestParseError as e:
                    # There was a problem parsing, unload ourselves in case they
                    # catch this error and try again later, we will show the
                    # correct error
                    self.Unload()
                    raise e

                if self.IsMirror:
                    self._AddMetaProjectMirror(self.repoProject)
                    self._AddMetaProjectMirror(self.manifestProject)

                self._loaded = True
            finally:
                if override:
                    self.manifestFile = savedManifestFile

            # Now that we have loaded this manifest, load any submanifests as
            # well. We need to do this after self._loaded is set to avoid
            # looping.
            for name in self._submanifests:
                tree = self._submanifests[name]
                tree.ToSubmanifestSpec()
                present = os.path.exists(
                    os.path.join(self.subdir, MANIFEST_FILE_NAME)
                )
                if present and tree.present and not tree.repo_client:
                    if initial_client and initial_client.topdir == self.topdir:
                        tree.repo_client = self
                        tree.present = present
                    elif not os.path.exists(self.subdir):
                        tree.present = False
                if present and tree.present:
                    tree.repo_client._Load(
                        initial_client=initial_client,
                        submanifest_depth=submanifest_depth + 1,
                    )

    def _ParseManifestXml(
        self,
        path,
        include_root,
        parent_groups="",
        restrict_includes=True,
        parent_node=None,
    ):
        """Parse a manifest XML and return the computed nodes.

        Args:
            path: The XML file to read & parse.
            include_root: The path to interpret include "name"s relative to.
            parent_groups: The groups to apply to this projects.
            restrict_includes: Whether to constrain the "name" attribute of
                includes.
            parent_node: The parent include node, to apply attribute to this
                projects.

        Returns:
            List of XML nodes.
        """
        try:
            root = xml.dom.minidom.parse(path)
        except (OSError, xml.parsers.expat.ExpatError) as e:
            raise ManifestParseError(f"error parsing manifest {path}: {e}")

        if not root or not root.childNodes:
            raise ManifestParseError(f"no root node in {path}")

        for manifest in root.childNodes:
            if (
                manifest.nodeType == manifest.ELEMENT_NODE
                and manifest.nodeName == "manifest"
            ):
                break
        else:
            raise ManifestParseError(f"no <manifest> in {path}")

        nodes = []
        for node in manifest.childNodes:
            if node.nodeName == "include":
                name = self._reqatt(node, "name")
                if restrict_includes:
                    msg = self._CheckLocalPath(name)
                    if msg:
                        raise ManifestInvalidPathError(
                            f'<include> invalid "name": {name}: {msg}'
                        )
                include_groups = ""
                if parent_groups:
                    include_groups = parent_groups
                if node.hasAttribute("groups"):
                    include_groups = (
                        node.getAttribute("groups") + "," + include_groups
                    )
                fp = os.path.join(include_root, name)
                if not os.path.isfile(fp):
                    raise ManifestParseError(
                        "include [%s/]%s doesn't exist or isn't a file"
                        % (include_root, name)
                    )
                try:
                    nodes.extend(
                        self._ParseManifestXml(
                            fp, include_root, include_groups, parent_node=node
                        )
                    )
                # should isolate this to the exact exception, but that's
                # tricky.  actual parsing implementation may vary.
                except (
                    KeyboardInterrupt,
                    RuntimeError,
                    SystemExit,
                    ManifestParseError,
                ):
                    raise
                except Exception as e:
                    raise ManifestParseError(
                        f"failed parsing included manifest {name}: {e}"
                    )
            else:
                if parent_groups and node.nodeName == "project":
                    nodeGroups = parent_groups
                    if node.hasAttribute("groups"):
                        nodeGroups = (
                            node.getAttribute("groups") + "," + nodeGroups
                        )
                    node.setAttribute("groups", nodeGroups)
                if (
                    parent_node
                    and node.nodeName == "project"
                    and not node.hasAttribute("revision")
                ):
                    node.setAttribute(
                        "revision", parent_node.getAttribute("revision")
                    )
                nodes.append(node)
        return nodes

    def _ParseManifest(self, node_list):
        for node in itertools.chain(*node_list):
            if node.nodeName == "remote":
                remote = self._ParseRemote(node)
                if remote:
                    if remote.name in self._remotes:
                        if remote != self._remotes[remote.name]:
                            raise ManifestParseError(
                                "remote %s already exists with different "
                                "attributes" % (remote.name)
                            )
                    else:
                        self._remotes[remote.name] = remote

        for node in itertools.chain(*node_list):
            if node.nodeName == "default":
                new_default = self._ParseDefault(node)
                emptyDefault = (
                    not node.hasAttributes() and not node.hasChildNodes()
                )
                if self._default is None:
                    self._default = new_default
                elif not emptyDefault and new_default != self._default:
                    raise ManifestParseError(
                        "duplicate default in %s" % (self.manifestFile)
                    )

        if self._default is None:
            self._default = _Default()

        submanifest_paths = set()
        for node in itertools.chain(*node_list):
            if node.nodeName == "submanifest":
                submanifest = self._ParseSubmanifest(node)
                if submanifest:
                    if submanifest.name in self._submanifests:
                        if submanifest != self._submanifests[submanifest.name]:
                            raise ManifestParseError(
                                "submanifest %s already exists with different "
                                "attributes" % (submanifest.name)
                            )
                    else:
                        self._submanifests[submanifest.name] = submanifest
                        submanifest_paths.add(submanifest.relpath)

        for node in itertools.chain(*node_list):
            if node.nodeName == "notice":
                if self._notice is not None:
                    raise ManifestParseError(
                        "duplicate notice in %s" % (self.manifestFile)
                    )
                self._notice = self._ParseNotice(node)

        for node in itertools.chain(*node_list):
            if node.nodeName == "manifest-server":
                url = self._reqatt(node, "url")
                if self._manifest_server is not None:
                    raise ManifestParseError(
                        "duplicate manifest-server in %s" % (self.manifestFile)
                    )
                self._manifest_server = url

        def recursively_add_projects(project):
            projects = self._projects.setdefault(project.name, [])
            if project.relpath is None:
                raise ManifestParseError(
                    "missing path for %s in %s"
                    % (project.name, self.manifestFile)
                )
            if project.relpath in self._paths:
                raise ManifestParseError(
                    "duplicate path %s in %s"
                    % (project.relpath, self.manifestFile)
                )
            for tree in submanifest_paths:
                if project.relpath.startswith(tree):
                    raise ManifestParseError(
                        "project %s conflicts with submanifest path %s"
                        % (project.relpath, tree)
                    )
            self._paths[project.relpath] = project
            projects.append(project)
            for subproject in project.subprojects:
                recursively_add_projects(subproject)

        repo_hooks_project = None
        enabled_repo_hooks = None
        failed_revision_changes = []
        for node in itertools.chain(*node_list):
            if node.nodeName == "project":
                project = self._ParseProject(node)
                recursively_add_projects(project)
            if node.nodeName == "extend-project":
                name = self._reqatt(node, "name")

                if name not in self._projects:
                    raise ManifestParseError(
                        "extend-project element specifies non-existent "
                        "project: %s" % name
                    )

                path = node.getAttribute("path")
                dest_path = node.getAttribute("dest-path")
                groups = node.getAttribute("groups")
                if groups:
                    groups = self._ParseList(groups)
                revision = node.getAttribute("revision")
                remote_name = node.getAttribute("remote")
                if not remote_name:
                    remote = self._default.remote
                else:
                    remote = self._get_remote(node)
                dest_branch = node.getAttribute("dest-branch")
                upstream = node.getAttribute("upstream")
                base_revision = node.getAttribute("base-rev")

                named_projects = self._projects[name]
                if dest_path and not path and len(named_projects) > 1:
                    raise ManifestParseError(
                        "extend-project cannot use dest-path when "
                        "matching multiple projects: %s" % name
                    )
                for p in self._projects[name]:
                    if path and p.relpath != path:
                        continue
                    if groups:
                        p.groups.extend(groups)
                    if revision:
                        if base_revision:
                            if p.revisionExpr != base_revision:
                                failed_revision_changes.append(
                                    "extend-project name %s mismatch base "
                                    "%s vs revision %s"
                                    % (name, base_revision, p.revisionExpr)
                                )
                        p.SetRevision(revision)

                    if remote_name:
                        p.remote = remote.ToRemoteSpec(name)
                    if dest_branch:
                        p.dest_branch = dest_branch
                    if upstream:
                        p.upstream = upstream

                    if dest_path:
                        del self._paths[p.relpath]
                        (
                            relpath,
                            worktree,
                            gitdir,
                            objdir,
                            _,
                        ) = self.GetProjectPaths(name, dest_path, remote.name)
                        p.UpdatePaths(relpath, worktree, gitdir, objdir)
                        self._paths[p.relpath] = p

            if node.nodeName == "repo-hooks":
                # Only one project can be the hooks project
                if repo_hooks_project is not None:
                    raise ManifestParseError(
                        "duplicate repo-hooks in %s" % (self.manifestFile)
                    )

                # Get the name of the project and the (space-separated) list of
                # enabled.
                repo_hooks_project = self._reqatt(node, "in-project")
                enabled_repo_hooks = self._ParseList(
                    self._reqatt(node, "enabled-list")
                )
            if node.nodeName == "superproject":
                name = self._reqatt(node, "name")
                # There can only be one superproject.
                if self._superproject:
                    raise ManifestParseError(
                        "duplicate superproject in %s" % (self.manifestFile)
                    )
                remote_name = node.getAttribute("remote")
                if not remote_name:
                    remote = self._default.remote
                else:
                    remote = self._get_remote(node)
                if remote is None:
                    raise ManifestParseError(
                        "no remote for superproject %s within %s"
                        % (name, self.manifestFile)
                    )
                revision = node.getAttribute("revision") or remote.revision
                if not revision:
                    revision = self._default.revisionExpr
                if not revision:
                    raise ManifestParseError(
                        "no revision for superproject %s within %s"
                        % (name, self.manifestFile)
                    )
                self._superproject = Superproject(
                    self,
                    name=name,
                    remote=remote.ToRemoteSpec(name),
                    revision=revision,
                )
            if node.nodeName == "contactinfo":
                bugurl = self._reqatt(node, "bugurl")
                # This element can be repeated, later entries will clobber
                # earlier ones.
                self._contactinfo = ContactInfo(bugurl)

            if node.nodeName == "remove-project":
                name = node.getAttribute("name")
                path = node.getAttribute("path")
                base_revision = node.getAttribute("base-rev")

                # Name or path needed.
                if not name and not path:
                    raise ManifestParseError(
                        "remove-project must have name and/or path"
                    )

                removed_project = ""

                # Find and remove projects based on name and/or path.
                for projname, projects in list(self._projects.items()):
                    for p in projects:
                        if name == projname and not path:
                            if base_revision:
                                if p.revisionExpr != base_revision:
                                    failed_revision_changes.append(
                                        "remove-project name %s mismatch base "
                                        "%s vs revision %s"
                                        % (name, base_revision, p.revisionExpr)
                                    )
                            del self._paths[p.relpath]
                            if not removed_project:
                                del self._projects[name]
                                removed_project = name
                        elif path == p.relpath and (
                            name == projname or not name
                        ):
                            if base_revision:
                                if p.revisionExpr != base_revision:
                                    failed_revision_changes.append(
                                        "remove-project path %s mismatch base "
                                        "%s vs revision %s"
                                        % (
                                            p.relpath,
                                            base_revision,
                                            p.revisionExpr,
                                        )
                                    )
                            self._projects[projname].remove(p)
                            del self._paths[p.relpath]
                            removed_project = p.name

                # If the manifest removes the hooks project, treat it as if
                # it deleted the repo-hooks element too.
                if (
                    removed_project
                    and removed_project not in self._projects
                    and repo_hooks_project == removed_project
                ):
                    repo_hooks_project = None

                if not removed_project and not XmlBool(node, "optional", False):
                    raise ManifestParseError(
                        "remove-project element specifies non-existent "
                        "project: %s" % node.toxml()
                    )

        if failed_revision_changes:
            raise ManifestParseError(
                "revision base check failed, rebase patches and update "
                "base revs for: ",
                failed_revision_changes,
            )

        # Store repo hooks project information.
        if repo_hooks_project:
            # Store a reference to the Project.
            try:
                repo_hooks_projects = self._projects[repo_hooks_project]
            except KeyError:
                raise ManifestParseError(
                    "project %s not found for repo-hooks" % (repo_hooks_project)
                )

            if len(repo_hooks_projects) != 1:
                raise ManifestParseError(
                    "internal error parsing repo-hooks in %s"
                    % (self.manifestFile)
                )
            self._repo_hooks_project = repo_hooks_projects[0]
            # Store the enabled hooks in the Project object.
            self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks

    def _AddMetaProjectMirror(self, m):
        name = None
        m_url = m.GetRemote().url
        if m_url.endswith("/.git"):
            raise ManifestParseError("refusing to mirror %s" % m_url)

        if self._default and self._default.remote:
            url = self._default.remote.resolvedFetchUrl
            if not url.endswith("/"):
                url += "/"
            if m_url.startswith(url):
                remote = self._default.remote
                name = m_url[len(url) :]

        if name is None:
            s = m_url.rindex("/") + 1
            manifestUrl = self.manifestProject.config.GetString(
                "remote.origin.url"
            )
            remote = _XmlRemote(
                "origin", fetch=m_url[:s], manifestUrl=manifestUrl
            )
            name = m_url[s:]

        if name.endswith(".git"):
            name = name[:-4]

        if name not in self._projects:
            m.PreSync()
            gitdir = os.path.join(self.topdir, "%s.git" % name)
            project = Project(
                manifest=self,
                name=name,
                remote=remote.ToRemoteSpec(name),
                gitdir=gitdir,
                objdir=gitdir,
                worktree=None,
                relpath=name or None,
                revisionExpr=m.revisionExpr,
                revisionId=None,
            )
            self._projects[project.name] = [project]
            self._paths[project.relpath] = project

    def _ParseRemote(self, node):
        """
        reads a <remote> element from the manifest file
        """
        name = self._reqatt(node, "name")
        alias = node.getAttribute("alias")
        if alias == "":
            alias = None
        fetch = self._reqatt(node, "fetch")
        pushUrl = node.getAttribute("pushurl")
        if pushUrl == "":
            pushUrl = None
        review = node.getAttribute("review")
        if review == "":
            review = None
        revision = node.getAttribute("revision")
        if revision == "":
            revision = None
        manifestUrl = self.manifestProject.config.GetString("remote.origin.url")

        remote = _XmlRemote(
            name, alias, fetch, pushUrl, manifestUrl, review, revision
        )

        for n in node.childNodes:
            if n.nodeName == "annotation":
                self._ParseAnnotation(remote, n)

        return remote

    def _ParseDefault(self, node):
        """
        reads a <default> element from the manifest file
        """
        d = _Default()
        d.remote = self._get_remote(node)
        d.revisionExpr = node.getAttribute("revision")
        if d.revisionExpr == "":
            d.revisionExpr = None

        d.destBranchExpr = node.getAttribute("dest-branch") or None
        d.upstreamExpr = node.getAttribute("upstream") or None

        d.sync_j = XmlInt(node, "sync-j", None)
        if d.sync_j is not None and d.sync_j <= 0:
            raise ManifestParseError(
                '%s: sync-j must be greater than 0, not "%s"'
                % (self.manifestFile, d.sync_j)
            )

        d.sync_c = XmlBool(node, "sync-c", False)
        d.sync_s = XmlBool(node, "sync-s", False)
        d.sync_tags = XmlBool(node, "sync-tags", True)
        return d

    def _ParseNotice(self, node):
        """
        reads a <notice> element from the manifest file

        The <notice> element is distinct from other tags in the XML in that the
        data is conveyed between the start and end tag (it's not an
        empty-element tag).

        The white space (carriage returns, indentation) for the notice element
        is relevant and is parsed in a way that is based on how python
        docstrings work. In fact, the code is remarkably similar to here:
          http://www.python.org/dev/peps/pep-0257/
        """
        # Get the data out of the node...
        notice = node.childNodes[0].data

        # Figure out minimum indentation, skipping the first line (the same line
        # as the <notice> tag)...
        minIndent = sys.maxsize
        lines = notice.splitlines()
        for line in lines[1:]:
            lstrippedLine = line.lstrip()
            if lstrippedLine:
                indent = len(line) - len(lstrippedLine)
                minIndent = min(indent, minIndent)

        # Strip leading / trailing blank lines and also indentation.
        cleanLines = [lines[0].strip()]
        for line in lines[1:]:
            cleanLines.append(line[minIndent:].rstrip())

        # Clear completely blank lines from front and back...
        while cleanLines and not cleanLines[0]:
            del cleanLines[0]
        while cleanLines and not cleanLines[-1]:
            del cleanLines[-1]

        return "\n".join(cleanLines)

    def _ParseSubmanifest(self, node):
        """Reads a <submanifest> element from the manifest file."""
        name = self._reqatt(node, "name")
        remote = node.getAttribute("remote")
        if remote == "":
            remote = None
        project = node.getAttribute("project")
        if project == "":
            project = None
        revision = node.getAttribute("revision")
        if revision == "":
            revision = None
        manifestName = node.getAttribute("manifest-name")
        if manifestName == "":
            manifestName = None
        groups = ""
        if node.hasAttribute("groups"):
            groups = node.getAttribute("groups")
        groups = self._ParseList(groups)
        default_groups = self._ParseList(node.getAttribute("default-groups"))
        path = node.getAttribute("path")
        if path == "":
            path = None
            if revision:
                msg = self._CheckLocalPath(revision.split("/")[-1])
                if msg:
                    raise ManifestInvalidPathError(
                        '<submanifest> invalid "revision": %s: %s'
                        % (revision, msg)
                    )
            else:
                msg = self._CheckLocalPath(name)
                if msg:
                    raise ManifestInvalidPathError(
                        f'<submanifest> invalid "name": {name}: {msg}'
                    )
        else:
            msg = self._CheckLocalPath(path)
            if msg:
                raise ManifestInvalidPathError(
                    f'<submanifest> invalid "path": {path}: {msg}'
                )

        submanifest = _XmlSubmanifest(
            name,
            remote,
            project,
            revision,
            manifestName,
            groups,
            default_groups,
            path,
            self,
        )

        for n in node.childNodes:
            if n.nodeName == "annotation":
                self._ParseAnnotation(submanifest, n)

        return submanifest

    def _JoinName(self, parent_name, name):
        return os.path.join(parent_name, name)

    def _UnjoinName(self, parent_name, name):
        return os.path.relpath(name, parent_name)

    def _ParseProject(self, node, parent=None, **extra_proj_attrs):
        """
        reads a <project> element from the manifest file
        """
        name = self._reqatt(node, "name")
        msg = self._CheckLocalPath(name, dir_ok=True)
        if msg:
            raise ManifestInvalidPathError(
                f'<project> invalid "name": {name}: {msg}'
            )
        if parent:
            name = self._JoinName(parent.name, name)

        remote = self._get_remote(node)
        if remote is None:
            remote = self._default.remote
        if remote is None:
            raise ManifestParseError(
                f"no remote for project {name} within {self.manifestFile}"
            )

        revisionExpr = node.getAttribute("revision") or remote.revision
        if not revisionExpr:
            revisionExpr = self._default.revisionExpr
        if not revisionExpr:
            raise ManifestParseError(
                "no revision for project %s within %s"
                % (name, self.manifestFile)
            )

        path = node.getAttribute("path")
        if not path:
            path = name
        else:
            # NB: The "." project is handled specially in
            # Project.Sync_LocalHalf.
            msg = self._CheckLocalPath(path, dir_ok=True, cwd_dot_ok=True)
            if msg:
                raise ManifestInvalidPathError(
                    f'<project> invalid "path": {path}: {msg}'
                )

        rebase = XmlBool(node, "rebase", True)
        sync_c = XmlBool(node, "sync-c", False)
        sync_s = XmlBool(node, "sync-s", self._default.sync_s)
        sync_tags = XmlBool(node, "sync-tags", self._default.sync_tags)

        clone_depth = XmlInt(node, "clone-depth")
        if clone_depth is not None and clone_depth <= 0:
            raise ManifestParseError(
                '%s: clone-depth must be greater than 0, not "%s"'
                % (self.manifestFile, clone_depth)
            )

        dest_branch = (
            node.getAttribute("dest-branch") or self._default.destBranchExpr
        )

        upstream = node.getAttribute("upstream") or self._default.upstreamExpr

        groups = ""
        if node.hasAttribute("groups"):
            groups = node.getAttribute("groups")
        groups = self._ParseList(groups)

        if parent is None:
            (
                relpath,
                worktree,
                gitdir,
                objdir,
                use_git_worktrees,
            ) = self.GetProjectPaths(name, path, remote.name)
        else:
            use_git_worktrees = False
            relpath, worktree, gitdir, objdir = self.GetSubprojectPaths(
                parent, name, path
            )

        default_groups = ["all", "name:%s" % name, "path:%s" % relpath]
        groups.extend(set(default_groups).difference(groups))

        if self.IsMirror and node.hasAttribute("force-path"):
            if XmlBool(node, "force-path", False):
                gitdir = os.path.join(self.topdir, "%s.git" % path)

        project = Project(
            manifest=self,
            name=name,
            remote=remote.ToRemoteSpec(name),
            gitdir=gitdir,
            objdir=objdir,
            worktree=worktree,
            relpath=relpath,
            revisionExpr=revisionExpr,
            revisionId=None,
            rebase=rebase,
            groups=groups,
            sync_c=sync_c,
            sync_s=sync_s,
            sync_tags=sync_tags,
            clone_depth=clone_depth,
            upstream=upstream,
            parent=parent,
            dest_branch=dest_branch,
            use_git_worktrees=use_git_worktrees,
            **extra_proj_attrs,
        )

        for n in node.childNodes:
            if n.nodeName == "copyfile":
                self._ParseCopyFile(project, n)
            if n.nodeName == "linkfile":
                self._ParseLinkFile(project, n)
            if n.nodeName == "annotation":
                self._ParseAnnotation(project, n)
            if n.nodeName == "project":
                project.subprojects.append(
                    self._ParseProject(n, parent=project)
                )

        return project

    def GetProjectPaths(self, name, path, remote):
        """Return the paths for a project.

        Args:
            name: a string, the name of the project.
            path: a string, the path of the project.
            remote: a string, the remote.name of the project.

        Returns:
            A tuple of (relpath, worktree, gitdir, objdir, use_git_worktrees)
            for the project with |name| and |path|.
        """
        # The manifest entries might have trailing slashes.  Normalize them to
        # avoid unexpected filesystem behavior since we do string concatenation
        # below.
        path = path.rstrip("/")
        name = name.rstrip("/")
        remote = remote.rstrip("/")
        use_git_worktrees = False
        use_remote_name = self.is_multimanifest
        relpath = path
        if self.IsMirror:
            worktree = None
            gitdir = os.path.join(self.topdir, "%s.git" % name)
            objdir = gitdir
        else:
            if use_remote_name:
                namepath = os.path.join(remote, f"{name}.git")
            else:
                namepath = f"{name}.git"
            worktree = os.path.join(self.topdir, path).replace("\\", "/")
            gitdir = os.path.join(self.subdir, "projects", "%s.git" % path)
            # We allow people to mix git worktrees & non-git worktrees for now.
            # This allows for in situ migration of repo clients.
            if os.path.exists(gitdir) or not self.UseGitWorktrees:
                objdir = os.path.join(self.repodir, "project-objects", namepath)
            else:
                use_git_worktrees = True
                gitdir = os.path.join(self.repodir, "worktrees", namepath)
                objdir = gitdir
        return relpath, worktree, gitdir, objdir, use_git_worktrees

    def GetProjectsWithName(self, name, all_manifests=False):
        """All projects with |name|.

        Args:
            name: a string, the name of the project.
            all_manifests: a boolean, if True, then all manifests are searched.
                If False, then only this manifest is searched.

        Returns:
            A list of Project instances with name |name|.
        """
        if all_manifests:
            return list(
                itertools.chain.from_iterable(
                    x._projects.get(name, []) for x in self.all_manifests
                )
            )
        return self._projects.get(name, [])

    def GetSubprojectName(self, parent, submodule_path):
        return os.path.join(parent.name, submodule_path)

    def _JoinRelpath(self, parent_relpath, relpath):
        return os.path.join(parent_relpath, relpath)

    def _UnjoinRelpath(self, parent_relpath, relpath):
        return os.path.relpath(relpath, parent_relpath)

    def GetSubprojectPaths(self, parent, name, path):
        # The manifest entries might have trailing slashes.  Normalize them to
        # avoid unexpected filesystem behavior since we do string concatenation
        # below.
        path = path.rstrip("/")
        name = name.rstrip("/")
        relpath = self._JoinRelpath(parent.relpath, path)
        subprojects = os.path.join(parent.gitdir, "subprojects", f"{path}.git")
        modules = os.path.join(parent.gitdir, "modules", path)
        if platform_utils.isdir(subprojects):
            gitdir = subprojects
        else:
            gitdir = modules
        objdir = os.path.join(
            parent.gitdir, "subproject-objects", "%s.git" % name
        )
        if self.IsMirror:
            worktree = None
        else:
            worktree = os.path.join(parent.worktree, path).replace("\\", "/")
        return relpath, worktree, gitdir, objdir

    @staticmethod
    def _CheckLocalPath(path, dir_ok=False, cwd_dot_ok=False):
        """Verify |path| is reasonable for use in filesystem paths.

        Used with <copyfile> & <linkfile> & <project> elements.

        This only validates the |path| in isolation: it does not check against
        the current filesystem state.  Thus it is suitable as a first-past in a
        parser.

        It enforces a number of constraints:
        * No empty paths.
        * No "~" in paths.
        * No Unicode codepoints that filesystems might elide when normalizing.
        * No relative path components like "." or "..".
        * No absolute paths.
        * No ".git" or ".repo*" path components.

        Args:
            path: The path name to validate.
            dir_ok: Whether |path| may force a directory (e.g. end in a /).
            cwd_dot_ok: Whether |path| may be just ".".

        Returns:
            None if |path| is OK, a failure message otherwise.
        """
        if not path:
            return "empty paths not allowed"

        if "~" in path:
            return "~ not allowed (due to 8.3 filenames on Windows filesystems)"

        path_codepoints = set(path)

        # Some filesystems (like Apple's HFS+) try to normalize Unicode
        # codepoints which means there are alternative names for ".git".  Reject
        # paths with these in it as there shouldn't be any reasonable need for
        # them here. The set of codepoints here was cribbed from jgit's
        # implementation:
        # https://eclipse.googlesource.com/jgit/jgit/+/9110037e3e9461ff4dac22fee84ef3694ed57648/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java#884
        BAD_CODEPOINTS = {
            "\u200C",  # ZERO WIDTH NON-JOINER
            "\u200D",  # ZERO WIDTH JOINER
            "\u200E",  # LEFT-TO-RIGHT MARK
            "\u200F",  # RIGHT-TO-LEFT MARK
            "\u202A",  # LEFT-TO-RIGHT EMBEDDING
            "\u202B",  # RIGHT-TO-LEFT EMBEDDING
            "\u202C",  # POP DIRECTIONAL FORMATTING
            "\u202D",  # LEFT-TO-RIGHT OVERRIDE
            "\u202E",  # RIGHT-TO-LEFT OVERRIDE
            "\u206A",  # INHIBIT SYMMETRIC SWAPPING
            "\u206B",  # ACTIVATE SYMMETRIC SWAPPING
            "\u206C",  # INHIBIT ARABIC FORM SHAPING
            "\u206D",  # ACTIVATE ARABIC FORM SHAPING
            "\u206E",  # NATIONAL DIGIT SHAPES
            "\u206F",  # NOMINAL DIGIT SHAPES
            "\uFEFF",  # ZERO WIDTH NO-BREAK SPACE
        }
        if BAD_CODEPOINTS & path_codepoints:
            # This message is more expansive than reality, but should be fine.
            return "Unicode combining characters not allowed"

        # Reject newlines as there shouldn't be any legitmate use for them,
        # they'll be confusing to users, and they can easily break tools that
        # expect to be able to iterate over newline delimited lists.  This even
        # applies to our own code like .repo/project.list.
        if {"\r", "\n"} & path_codepoints:
            return "Newlines not allowed"

        # Assume paths might be used on case-insensitive filesystems.
        path = path.lower()

        # Split up the path by its components.  We can't use os.path.sep
        # exclusively as some platforms (like Windows) will convert / to \ and
        # that bypasses all our constructed logic here.  Especially since
        # manifest authors only use / in their paths.
        resep = re.compile(r"[/%s]" % re.escape(os.path.sep))
        # Strip off trailing slashes as those only produce '' elements, and we
        # use parts to look for individual bad components.
        parts = resep.split(path.rstrip("/"))

        # Some people use src="." to create stable links to projects.  Lets
        # allow that but reject all other uses of "." to keep things simple.
        if not cwd_dot_ok or parts != ["."]:
            for part in set(parts):
                if part in {".", "..", ".git"} or part.startswith(".repo"):
                    return f"bad component: {part}"

        if not dir_ok and resep.match(path[-1]):
            return "dirs not allowed"

        # NB: The two abspath checks here are to handle platforms with multiple
        # filesystem path styles (e.g. Windows).
        norm = os.path.normpath(path)
        if (
            norm == ".."
            or (
                len(norm) >= 3
                and norm.startswith("..")
                and resep.match(norm[0])
            )
            or os.path.isabs(norm)
            or norm.startswith("/")
        ):
            return "path cannot be outside"

    @classmethod
    def _ValidateFilePaths(cls, element, src, dest):
        """Verify |src| & |dest| are reasonable for <copyfile> & <linkfile>.

        We verify the path independent of any filesystem state as we won't have
        a checkout available to compare to.  i.e. This is for parsing validation
        purposes only.

        We'll do full/live sanity checking before we do the actual filesystem
        modifications in _CopyFile/_LinkFile/etc...
        """
        # |dest| is the file we write to or symlink we create.
        # It is relative to the top of the repo client checkout.
        msg = cls._CheckLocalPath(dest)
        if msg:
            raise ManifestInvalidPathError(
                f'<{element}> invalid "dest": {dest}: {msg}'
            )

        # |src| is the file we read from or path we point to for symlinks.
        # It is relative to the top of the git project checkout.
        is_linkfile = element == "linkfile"
        msg = cls._CheckLocalPath(
            src, dir_ok=is_linkfile, cwd_dot_ok=is_linkfile
        )
        if msg:
            raise ManifestInvalidPathError(
                f'<{element}> invalid "src": {src}: {msg}'
            )

    def _ParseCopyFile(self, project, node):
        src = self._reqatt(node, "src")
        dest = self._reqatt(node, "dest")
        if not self.IsMirror:
            # src is project relative;
            # dest is relative to the top of the tree.
            # We only validate paths if we actually plan to process them.
            self._ValidateFilePaths("copyfile", src, dest)
            project.AddCopyFile(src, dest, self.topdir)

    def _ParseLinkFile(self, project, node):
        src = self._reqatt(node, "src")
        dest = self._reqatt(node, "dest")
        if not self.IsMirror:
            # src is project relative;
            # dest is relative to the top of the tree.
            # We only validate paths if we actually plan to process them.
            self._ValidateFilePaths("linkfile", src, dest)
            project.AddLinkFile(src, dest, self.topdir)

    def _ParseAnnotation(self, element, node):
        name = self._reqatt(node, "name")
        value = self._reqatt(node, "value")
        try:
            keep = self._reqatt(node, "keep").lower()
        except ManifestParseError:
            keep = "true"
        if keep != "true" and keep != "false":
            raise ManifestParseError(
                'optional "keep" attribute must be ' '"true" or "false"'
            )
        element.AddAnnotation(name, value, keep)

    def _get_remote(self, node):
        name = node.getAttribute("remote")
        if not name:
            return None

        v = self._remotes.get(name)
        if not v:
            raise ManifestParseError(
                f"remote {name} not defined in {self.manifestFile}"
            )
        return v

    def _reqatt(self, node, attname):
        """
        reads a required attribute from the node.
        """
        v = node.getAttribute(attname)
        if not v:
            raise ManifestParseError(
                "no %s in <%s> within %s"
                % (attname, node.nodeName, self.manifestFile)
            )
        return v

    def projectsDiff(self, manifest):
        """return the projects differences between two manifests.

        The diff will be from self to given manifest.

        """
        fromProjects = self.paths
        toProjects = manifest.paths

        fromKeys = sorted(fromProjects.keys())
        toKeys = set(toProjects.keys())

        diff = {
            "added": [],
            "removed": [],
            "missing": [],
            "changed": [],
            "unreachable": [],
        }

        for proj in fromKeys:
            fromProj = fromProjects[proj]
            if proj not in toKeys:
                diff["removed"].append(fromProj)
            elif not fromProj.Exists:
                diff["missing"].append(toProjects[proj])
                toKeys.remove(proj)
            else:
                toProj = toProjects[proj]
                try:
                    fromRevId = fromProj.GetCommitRevisionId()
                    toRevId = toProj.GetCommitRevisionId()
                except ManifestInvalidRevisionError:
                    diff["unreachable"].append((fromProj, toProj))
                else:
                    if fromRevId != toRevId:
                        diff["changed"].append((fromProj, toProj))
                toKeys.remove(proj)

        diff["added"].extend(toProjects[proj] for proj in sorted(toKeys))

        return diff


class RepoClient(XmlManifest):
    """Manages a repo client checkout."""

    def __init__(
        self, repodir, manifest_file=None, submanifest_path="", **kwargs
    ):
        """Initialize.

        Args:
            repodir: Path to the .repo/ dir for holding all internal checkout
                state. It must be in the top directory of the repo client
                checkout.
            manifest_file: Full path to the manifest file to parse.  This will
                usually be |repodir|/|MANIFEST_FILE_NAME|.
            submanifest_path: The submanifest root relative to the repo root.
            **kwargs: Additional keyword arguments, passed to XmlManifest.
        """
        submanifest_path = submanifest_path or ""
        if submanifest_path:
            self._CheckLocalPath(submanifest_path)
            prefix = os.path.join(repodir, SUBMANIFEST_DIR, submanifest_path)
        else:
            prefix = repodir

        if os.path.exists(os.path.join(prefix, LOCAL_MANIFEST_NAME)):
            print(
                "error: %s is not supported; put local manifests in `%s` "
                "instead"
                % (
                    LOCAL_MANIFEST_NAME,
                    os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME),
                ),
                file=sys.stderr,
            )
            sys.exit(1)

        if manifest_file is None:
            manifest_file = os.path.join(prefix, MANIFEST_FILE_NAME)
        local_manifests = os.path.abspath(
            os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)
        )
        super().__init__(
            repodir,
            manifest_file,
            local_manifests,
            submanifest_path=submanifest_path,
            **kwargs,
        )

        # TODO: Completely separate manifest logic out of the client.
        self.manifest = self