"""
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.


This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
See the GNU General Public License for more details.


You should have received a copy of the GNU General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.

Copyright © 2019 Cloud Linux Software Inc.

This software is also available under ImunifyAV commercial license,
see <https://www.imunify360.com/legal/eula>

No file a cleanup run created outlives the run.

Part of the cleanup_precreate_report feature flag (see report_file): with it
off no journal is written and nothing here runs for a cleanup; the run's
files go with their own context managers, as before.

The agent removes the files it hands to the cleaner when the run ends. A
cleaner killed in the middle of an atomic write (rustcurator writes a temp
file next to its report or progress file, then renames it) leaves that temp
file in the temp dir; an agent that dies in the middle of a run leaves
everything.

So before the cleaner starts, a journal records the run: the files handed
over, and which of them is the report root made for the account. When the run ends the journal drives the sweep and is removed; a
journal still on disk later belongs to a run the agent did not see end, and
the next agent start (or cleanup tick, once its cleaner is gone) sweeps it.
Only the names the journal lists are removed, and the temp files named after
them, which carry the run's random token; the temp dir is listed for those
only after a cleaner that did not exit 0 and for an orphaned run. Nothing goes for a name pattern
alone: rustcurator 33.4/34.0 names its temp file .tmpXXXXXX, which outside
CageFS may as well be a file of the account's own, so one such cleaner killed
mid-write leaves that file behind.
"""

import errno
import json
import logging
import os
import re
import stat
import tempfile
import time
from contextlib import suppress
from pathlib import Path
from typing import Iterable, Optional

from defence360agent.subsys.persistent_state import PERSISTENT_STATE_DIR

logger = logging.getLogger(__name__)

JOURNAL_DIR = PERSISTENT_STATE_DIR / "cleanup_runs"
# later rustcurator names its atomic-write temp file .<file name>.<6 random>
_BESIDE_SUFFIX = re.compile(r"[0-9A-Za-z]{6}")
# a journal still being written by a run of this agent is younger than this
_UNFINISHED_JOURNAL_AGE = 60
_DIR_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
_ACL_XATTR = "system.posix_acl_access"
# the deferred sweeps still pending (see RunJournal.finish_after)
_late_sweeps = set()


def _process_start(pid: int) -> Optional[str]:
    try:
        with open("/proc/%d/stat" % pid, "rb") as stat_file:
            fields = stat_file.read().rsplit(b")", 1)[1].split()
    except (OSError, IndexError):
        return None
    # field 22 (starttime); fields[0] here is field 3 (state)
    return fields[19].decode() if len(fields) > 19 else None


def _open_dir(path: str) -> int:
    """Open the absolute *path* one component at a time from /, each
    relative to the one before and without following a symlink, so no
    renamed or replaced ancestor can lead elsewhere."""
    fd = os.open("/", _DIR_FLAGS)
    try:
        for part in path.split("/"):
            if part in ("", "."):
                continue
            if part == "..":
                raise OSError(errno.EINVAL, "Parent component", path)
            next_fd = os.open(part, _DIR_FLAGS, dir_fd=fd)
            os.close(fd)
            fd = next_fd
    except BaseException:
        os.close(fd)
        raise
    return fd


def _trusted(info, uid) -> bool:
    return info.st_uid in (0, uid)


def _resolve_trusted(path: str) -> Optional[str]:
    """*path* resolved like realpath, but following only a symlink root (or
    this process) owns in a directory no one else may replace it in; None
    when any component is a symlink, or a directory, someone else controls
    (another owner, or writable by others without the sticky bit)."""
    uid = os.geteuid()
    pending = [part for part in path.split("/") if part not in ("", ".")]
    current, hops = "/", 0
    try:
        while pending:
            part = pending.pop(0)
            if part == "..":
                current = os.path.dirname(current)
                continue
            candidate = os.path.join(current, part)
            info = os.lstat(candidate)
            if stat.S_ISLNK(info.st_mode):
                parent = os.lstat(current)
                hops += 1
                if (
                    hops > 40
                    or not _trusted(info, uid)
                    or not _trusted(parent, uid)
                    or (parent.st_mode & 0o022 and not parent.st_mode & 0o1000)
                ):
                    return None
                target = os.readlink(candidate)
                if target.startswith("/"):
                    current = "/"
                pending = [
                    p for p in target.split("/") if p not in ("", ".")
                ] + pending
                continue
            writable_by_others = info.st_mode & 0o022 and not (
                info.st_mode & 0o1000
            )
            if (
                not stat.S_ISDIR(info.st_mode)
                or not _trusted(info, uid)
                or writable_by_others
            ):
                return None
            current = candidate
    except OSError:
        return None
    return current


def _dir_id(path: str) -> Optional[list]:
    """Device and inode of the directory *path* names, opened as above."""
    try:
        fd = _open_dir(path)
    except OSError:
        return None
    try:
        info = os.fstat(fd)
        return [info.st_dev, info.st_ino]
    finally:
        os.close(fd)


def open_trusted_dir(path: str) -> int:
    """A descriptor of the directory *path* names, resolved only through
    symlinks and directories no account controls (see _resolve_trusted)
    and opened component by component without following a symlink."""
    resolved = _resolve_trusted(path)
    if resolved is None:
        raise OSError(errno.EPERM, "Not a directory only root controls", path)
    return _open_dir(resolved)


def _beside_temps(dir_fd: int, directory: str, names) -> bool:
    """Remove the atomic-write temp files named after the run's files: the
    name carries the run's random token, so it is the run's own. False when
    one could not be looked for or removed."""
    prefixes = ["." + name + "." for name in names]
    done = True
    try:
        # one entry at a time: a shared /tmp may hold a very large number
        with os.scandir(dir_fd) as entries:
            for entry in entries:
                name = entry.name
                prefix = next(
                    (p for p in prefixes if name.startswith(p)), None
                )
                if prefix is None or not _BESIDE_SUFFIX.fullmatch(
                    name[len(prefix) :]
                ):
                    continue
                if not _remove_temp(dir_fd, directory, name):
                    done = False
    except OSError as exc:
        logger.warning("Cannot read %s: %s", directory, exc)
        return False
    return done


def _remove_temp(dir_fd: int, directory: str, name: str) -> bool:
    try:
        info = os.stat(name, dir_fd=dir_fd, follow_symlinks=False)
        if stat.S_ISREG(info.st_mode) and info.st_nlink == 1:
            os.unlink(name, dir_fd=dir_fd)
            logger.info("Removed %s/%s the cleaner left", directory, name)
    except FileNotFoundError:
        pass
    except OSError as exc:
        logger.warning("Cannot remove %s/%s: %s", directory, name, exc)
        return False
    return True


def seal(fd: int) -> None:
    """Leave the open file *fd* to root alone, and empty: no ACL entry,
    mode 0600, no data. A descriptor the account opened before keeps its
    access, but to an empty file it cannot reopen."""
    with suppress(OSError):
        os.removexattr(fd, _ACL_XATTR)
    os.fchmod(fd, 0o600)
    os.ftruncate(fd, 0)


def _has_acl(fd: int) -> bool:
    try:
        os.getxattr(fd, _ACL_XATTR)
    except OSError:
        return False
    return True


def _seal_report(dir_fd: int, directory: str, name: str, report_id) -> None:
    """Seal *name* if it is still the report root made for the account
    (see report_file): a hard link the account made to it keeps nothing it
    can write once the name is removed. That is the recorded inode, owned
    by root and still carrying an ACL; a freed inode number a filesystem
    gave another file carries none."""
    try:
        fd = os.open(
            name,
            os.O_WRONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
            dir_fd=dir_fd,
        )
    except OSError as exc:
        logger.warning("Cannot seal %s/%s: %s", directory, name, exc)
        return
    try:
        info = os.fstat(fd)
        if (
            stat.S_ISREG(info.st_mode)
            and info.st_uid == os.geteuid()
            and [info.st_dev, info.st_ino] == report_id
            and _has_acl(fd)
        ):
            seal(fd)
    except OSError as exc:
        logger.warning("Cannot seal %s/%s: %s", directory, name, exc)
    finally:
        os.close(fd)


def _names_in(directory: str, paths) -> list:
    """Basenames of the *paths* that are entries of *directory*; any other
    path is not the run's and is left alone."""
    names = []
    for path in paths:
        parent, _, name = path.rpartition("/")
        if (
            name in ("", ".", "..")
            or os.path.normpath(parent or "/") != directory
        ):
            logger.warning(
                "Not removing %s: outside the run's temp dir %s",
                path,
                directory,
            )
            continue
        names.append(name)
    return names


def sweep(record: dict, beside: bool = True) -> bool:
    """Remove what the run *record* names and, if *beside*, the temp files
    named after it (the temp dir is listed for those: a cleaner that exited
    cleanly renamed them all); other keys (an older agent's journal) are
    ignored. True when nothing of it is left (a file already gone counts as
    removed).

    Every stat and unlink is relative to the run's temp dir, opened once
    component by component without following a symlink and checked to be
    the very directory (device, inode) the run started with, and acts on
    basenames only: whatever a path names meanwhile, nothing outside that
    directory goes. A run without that pin is not swept. The report root
    made for the account is sealed before it goes, if it is that very file.
    """
    files = [path for path in record.get("files", ()) if isinstance(path, str)]
    tempdir = record.get("tempdir")
    if not files:
        return True
    if not isinstance(tempdir, str) or not tempdir.startswith("/"):
        logger.warning("Not removing %s: the run names no temp dir", files)
        return True
    tempdir = os.path.normpath(tempdir)
    names = _names_in(tempdir, files)
    try:
        dir_fd = _open_dir(tempdir)
    except FileNotFoundError:
        return True
    except OSError as exc:
        if exc.errno in (errno.ELOOP, errno.ENOTDIR, errno.EINVAL):
            # replaced since the run started: what it names is not the run's
            logger.warning(
                "Not removing the files of a run in %s: no longer a directory",
                tempdir,
            )
            return True
        logger.warning("Cannot open %s: %s", tempdir, exc)
        return False
    expected = record.get("tempdir_id")
    if not (isinstance(expected, list) and len(expected) == 2):
        os.close(dir_fd)
        logger.warning(
            "Not removing the files of a run in %s: its directory was not"
            " pinned when the run started",
            tempdir,
        )
        return True
    info = os.fstat(dir_fd)
    if [info.st_dev, info.st_ino] != expected:
        os.close(dir_fd)
        logger.warning(
            "Not removing the files of a run in %s: no longer the run's"
            " directory",
            tempdir,
        )
        return True
    report_id = record.get("report_id")
    try:
        done = _beside_temps(dir_fd, tempdir, names) if beside else True
        for name in names:
            try:
                info = os.stat(name, dir_fd=dir_fd, follow_symlinks=False)
                if stat.S_ISDIR(info.st_mode):
                    continue
                if (
                    stat.S_ISREG(info.st_mode)
                    and [info.st_dev, info.st_ino] == report_id
                ):
                    _seal_report(dir_fd, tempdir, name, report_id)
                os.unlink(name, dir_fd=dir_fd)
            except FileNotFoundError:
                continue  # removed already, by the cleaner or the agent
            except OSError as exc:
                logger.warning("Cannot remove %s/%s: %s", tempdir, name, exc)
                done = False
        return done
    finally:
        os.close(dir_fd)


def _rebased(path: str, tempdir: str, real: str) -> str:
    parent, _, name = path.rpartition("/")
    if os.path.normpath(parent) == os.path.normpath(tempdir):
        return os.path.join(real, name)
    return path


class RunJournal:
    """Context manager around one cleaner run; see the module docstring."""

    def __init__(self, tempdir: str, files: Iterable):
        # resolved now, while no account has had a run file to play with,
        # and only through symlinks no account controls (some hosts link
        # /tmp); otherwise the path is kept as it is. Either way the sweep
        # acts only in this very directory: none if the walk to it meets a
        # symlink
        resolved = _resolve_trusted(tempdir)
        real = resolved or os.path.normpath(tempdir)
        self.record = {
            "pid": os.getpid(),
            "pid_start": _process_start(os.getpid()),
            "tempdir": real,
            "tempdir_id": _dir_id(real),
            "files": [_rebased(str(f), tempdir, real) for f in files],
        }
        self._path = None
        self._deferred = False
        self._beside = True  # until the cleaner exits cleanly

    def __enter__(self):
        self.record["started"] = time.time()
        self._write()
        return self

    def _write(self) -> None:
        try:
            JOURNAL_DIR.mkdir(mode=0o700, parents=True, exist_ok=True)
            with tempfile.NamedTemporaryFile(
                "w", dir=JOURNAL_DIR, suffix=".tmp", delete=False
            ) as journal:
                json.dump(self.record, journal)
            path = self._path or (
                JOURNAL_DIR / (Path(journal.name).stem + ".json")
            )
            os.replace(journal.name, path)
            self._path = path
        except OSError as exc:
            # the sweep at the end of the run still happens
            logger.warning("Cannot record the cleanup run: %s", exc)

    def note_file(self, tempdir: str, path: str) -> None:
        """Include an additional input before it is written or handed over."""
        self.record["files"].append(
            _rebased(path, tempdir, self.record["tempdir"])
        )
        if self._path is not None:
            self._write()

    def note_cleaner(self, pid) -> None:
        """The agent stops without its children: while this cleaner runs,
        a later sweep leaves the run alone."""
        if type(pid) is not int:
            return
        self.record["cleaner_pid"] = pid
        self.record["cleaner_start"] = _process_start(pid)
        if self._path is not None:
            self._write()

    def note_report(self, identity) -> None:
        """Root made the report for the account (see report_file): the
        sweep seals that very file before it removes it, also after an
        agent crash, when release() never runs."""
        self.record["report_id"] = list(identity)
        if self._path is not None:
            self._write()

    def note_exit(self, returncode) -> None:
        """A cleaner that exited 0 renamed its temp files: its run's sweep
        does not list the temp dir for them."""
        self._beside = returncode != 0

    def finish(self) -> None:
        if not sweep(self.record, beside=self._beside):
            # kept for a retry at the next cleanup tick or agent start, which
            # may sweep a finished run while this agent is still alive
            self.record["finished"] = True
            if self._path is not None:
                self._write()
            return
        if self._path is not None:
            with suppress(OSError):
                self._path.unlink()
            self._path = None

    def finish_after(self, loop, awaitable) -> None:
        """The cleaner may still write after the run is left: sweep again,
        and drop the journal, once *awaitable* (its exit) is done. If that
        never happens here, the next agent start sweeps the run."""
        self._deferred = True

        async def finish_later():
            try:
                await awaitable
            except Exception:
                logger.exception("Cleanup run: did not see the cleaner exit")
            try:
                await loop.run_in_executor(None, self.finish)
            except Exception:
                logger.exception("Cannot remove the files of a cleanup run")

        task = loop.create_task(finish_later())
        _late_sweeps.add(task)
        task.add_done_callback(_late_sweeps.discard)

    def close(self) -> None:
        if self._deferred:
            # the journal stays for finish_later, which looks beside
            sweep(self.record, beside=False)
        else:
            self.finish()

    def __exit__(self, *_):
        self.close()


def _alive(record, pid_key, start_key) -> bool:
    """The same pid and start time: that process is still running."""
    pid, start = record.get(pid_key), record.get(start_key)
    return (
        type(pid) is int
        and start is not None
        and (_process_start(pid) == start)
    )


def sweep_orphaned_runs() -> int:
    """Sweep every run whose agent and cleaner are both gone (a previous
    agent recorded it and did not see it end), or that ended with files it
    could not remove; the number swept. Called at agent start and on every
    cleanup tick, for cleaners that outlive it; a journal goes only once its
    run left nothing."""
    swept = 0
    try:
        journals = sorted(JOURNAL_DIR.glob("*"))
    except OSError:
        return swept
    for path in journals:
        try:
            if path.suffix != ".json":
                # a journal write that never completed
                if (
                    time.time() - path.stat().st_mtime
                    > _UNFINISHED_JOURNAL_AGE
                ):
                    path.unlink()
                continue
            data = path.read_bytes()
        except OSError as exc:
            # tells nothing about the run: a later tick tries again
            logger.warning("Cannot read cleanup journal %s: %s", path, exc)
            continue
        try:
            record = json.loads(data)
        except ValueError as exc:
            logger.warning(
                "Dropping unreadable cleanup journal %s: %s", path, exc
            )
            with suppress(OSError):
                path.unlink()
            continue
        if not isinstance(record, dict):
            with suppress(OSError):
                path.unlink()
            continue
        agent_running = not record.get("finished") and _alive(
            record, "pid", "pid_start"
        )
        if agent_running or _alive(record, "cleaner_pid", "cleaner_start"):
            continue
        if not sweep(record):
            continue  # tried again on the next tick
        with suppress(OSError):
            path.unlink()
        swept += 1
    if swept:
        logger.info(
            "Removed the files of %d interrupted cleanup run(s)", swept
        )
    return swept
