"""
This program is free software: you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License,
or (at your option) any later version.


This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
See the GNU General Public License for more details.


You should have received a copy of the GNU General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.

Copyright © 2019 Cloud Linux Software Inc.

This software is also available under ImunifyAV commercial license,
see <https://www.imunify360.com/legal/eula>
"""
import asyncio
import heapq
import pwd
import os
import re
import shutil
import time
import threading
import uuid
from collections import defaultdict
from contextlib import suppress
from functools import partial
from logging import getLogger
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Tuple

from peewee import Case, fn

from defence360agent import utils
from defence360agent.utils import safe_fileops
from defence360agent.internals import feature_flags
from imav.malwarelib.cleanup import curator_version
from defence360agent.api import inactivity
from defence360agent.contracts.config import (
    Malware as Config,
)
from defence360agent.contracts.config import (
    MalwareTune,
    MyImunifyConfig,
    UserType,
)
from defence360agent.contracts.hook_events import HookEvent
from defence360agent.contracts.license import LicenseCLN
from defence360agent.contracts.messages import MessageType
from defence360agent.contracts.permissions import myimunify_protection_enabled
from defence360agent.contracts.plugins import (
    MessageSink,
    MessageSource,
    expect,
)
from defence360agent.internals.global_scope import g
from defence360agent.model import tls_check
from defence360agent.subsys.persistent_state import register_lock_file
from defence360agent.utils import (
    execute_iterable_expression,
    get_results_iterable_expression,
    Scope,
    is_system_user,
    nice_iterator,
    recurring_check,
    safe_cancel_task,
    split_for_chunk,
)
from defence360agent.utils.check_lock import check_lock
from defence360agent.utils.common import DAY, MINUTE, rate_limit
from imav.contracts.messages import MalwareDatabaseRestoreTask
from imav.malwarelib.cleanup import blacklist_hashes, sa_verdicts
from imav.malwarelib.cleanup.cleaner import (
    CleanupResult,
    MalwareCleaner,
    MalwareCleanupProxy,
)
from imav.malwarelib.cleanup import run_as
from imav.malwarelib.cleanup.leftovers import sweep_orphaned_runs
from imav.malwarelib.cleanup.reasons import local_details, log_diagnostic
from imav.malwarelib.cleanup.storage import CleanupStorage
from imav.malwarelib.config import (
    FAILED_TO_RESTORE_ORIGINAL,
    PARTIALLY_RESTORED_ORIGINAL,
    RESTORED_ORIGINAL,
    MalwareHitStatus,
    MalwareScanResourceType,
    MalwareScanType,
)
from imav.malwarelib.hit_removed_hook import (
    EXPIRED,
    delete_and_report,
    hooked,
)
from imav.malwarelib import engine, store_db
from imav.malwarelib.cleanup import RUSTCURATOR_BINARY
from imav.malwarelib.model import MalwareHistory, MalwareHit, MDSRestoreBatch
from imav.malwarelib.scan import ScanAlreadyCompleteError
from imav.malwarelib.scan.crontab import is_crontab
from imav.malwarelib.scan.mds import operations
from imav.malwarelib.scan.mds.cleaner import MalwareDatabaseCleaner
from imav.malwarelib.scan.mds.detached import (
    MDSDetachedCleanup,
    MDSDetachedRestore,
    MDSDetachedRestoreDir,
)
from imav.malwarelib.scan.mds.restore import (
    MalwareDatabaseRestore,
    guarded_route_available,
)
from defence360agent.subsys.panels.hosting_panel import HostingPanel
from imav.malwarelib.stop_process import cron as stop_cron
from imav.malwarelib.stop_process import mode as stop_mode
from imav.malwarelib.stop_process import registry as stop_registry
from imav.malwarelib.stop_process import proc as stop_proc
from imav.malwarelib.stop_process import report as stop_report
from imav.malwarelib.stop_process.controller import StopProcessController
from imav.malwarelib.stop_process.registry import release_orphans
from imav.malwarelib.stop_process.stopper import (
    COULD_NOT_FREEZE,
    FILE_NOT_CLEANED,
    FREEZE_TIMEOUT,
    READ_FAILED,
    RESPAWNING,
    STOP_CAP,
    cleaner_outcome,
)
from imav.malwarelib.subsys.malware import HackerTrapHitsSaver, MalwareAction
from imav.malwarelib.tenant_path import gather_exists, safe_exists
from imav.malwarelib.utils import malware_response
from imav.malwarelib.utils.user_list import (
    get_username_by_uid,
    is_uid,
)
from imav.malwarelib.model import MalwareIgnorePath

logger = getLogger(__name__)

#: seconds a small cleaner pass may still finish once the agent stops;
#: then its curator is terminated, as released cleanup does on cancel
SMALL_PASS_GRACE = 2.0
#: the owner's other detected files looked at before one of its processes
#: is stopped (newest first)
MAX_VETO_HITS = 10_000
#: seconds between two looks at the cron store's pending re-enables
CRON_RETRY_INTERVAL = 60.0
#: how often an idle tick looks for runs whose cleaner outlived the agent
RUNS_RETRY_INTERVAL = 60.0
_NOT_CLEANED_YET = (
    MalwareHitStatus.FOUND,
    MalwareHitStatus.CLEANUP_STARTED,
    MalwareHitStatus.CLEANUP_REQUIRES_MYIMUNIFY_PROTECTION,
    MalwareHitStatus.RESTORE_FROM_BACKUP_STARTED,
)


def _other_detected_hits(users, batch_ids) -> List[MalwareHit]:
    """The file hits of these accounts (the indexed home owner) detected
    and not cleaned yet, outside the batch: a process linked to one of
    them is never stopped. Runs in the executor."""
    with suppress(tls_check.OverridingReset):
        tls_check.reset()
    rows = [
        hit
        for hit in MalwareHit.select(
            MalwareHit.id,
            MalwareHit.orig_file,
            MalwareHit.user,
            MalwareHit.owner,
            MalwareHit.file_dev,
            MalwareHit.file_ino,
            MalwareHit.file_uid,
        )
        .where(
            MalwareHit.user.in_(sorted(users)),
            MalwareHit.resource_type == MalwareScanResourceType.FILE.value,
            MalwareHit.status.in_(_NOT_CLEANED_YET),
        )
        .order_by(MalwareHit.id.desc())
        .limit(MAX_VETO_HITS + len(batch_ids) + 1)
        if hit.id not in batch_ids
    ]
    if len(rows) > MAX_VETO_HITS:
        logger.info(
            "More than %d detected files of %s are not cleaned yet: the"
            " oldest are not checked before a process is stopped",
            MAX_VETO_HITS,
            ", ".join(sorted(users)),
        )
    return rows[:MAX_VETO_HITS]


# Deferred missing-file evidence must not prevent the agent going idle.
MAX_GONE_STOP_ATTEMPTS = 5

COUNT_OF_ATTEMPTS_TO_CLEANUP_PER_DAY = 4
# A path we keep cleaning successfully still burns a full backup copy on
# every failed round, so the reset above needs a ceiling of its own.
COUNT_OF_FAILURES_TO_CLEANUP_PER_DAY = 16
RESCAN_ATTEMPTS = 2
LOCK_FILE = register_lock_file("cleanup", Scope.AV_IM360)


def _record_stop_history(out, hits, cause, initiator):
    """Write one account's rows in a worker-owned DB connection."""
    with suppress(tls_check.OverridingReset):
        tls_check.reset()
    database = MalwareHistory._meta.database
    with database.connection_context():
        stop_report.record_locally(out, hits, cause=cause, initiator=initiator)


# accounts web servers execute client code as: www-data, apache, EL7 nobody.
# They stay cleanable on our side; the released cleaner still refuses 48/99
# itself until its guard adopts the same set, so until then they behave as
# before this guard existed.
WEB_EXEC_UIDS = frozenset((33, 48, 99))


def _system_owned(uid: int) -> bool:
    return uid not in WEB_EXEC_UIDS and is_system_user(uid)


def _system_owned_text(owner: str) -> bool:
    # Preserve int() syntax (Unicode digits, whitespace, signs, underscores)
    # that SQLite CAST does not accept. Called only for non-canonical text.
    return is_uid(owner) and _system_owned(int(owner))


_system_owner_connection = threading.local()


def _register_system_owned_text(database):
    connection = database.connection()
    if getattr(_system_owner_connection, "value", None) is not connection:
        # Each thread/reconnect has its own SQLite functions. Install only our
        # new function: re-registering existing functions fails with an active
        # cursor. Keep only the last connection per thread, not a growing cache.
        connection.create_function(
            "imav_system_owned_text", 1, _system_owned_text
        )
        _system_owner_connection.value = connection


def _root_cleanup_candidate(hit, capable=False, *, curator=None) -> bool:
    """Cheap scope checks before scheduling the database count."""
    if (
        hit.owner != "0"
        or not feature_flags.is_enabled("clean_system_owned_site_files")
        or (engine.curator_path() if curator is None else curator)
        != RUSTCURATOR_BINARY
        or not capable
        or not getattr(safe_fileops, "STRICT_COPY_BOUNDARY", False)
        or not hit.hash
    ):
        return False
    # Check the stored spelling before pathlib can collapse dot components.
    if any(part in (".", "..") for part in hit.orig_file.split("/")):
        return False
    path = str(hit.orig_file_path)
    if not re.fullmatch(r"/(?:home[^/]*/[^/]+|var/www/vhosts/[^/]+)/.+", path):
        return False
    name = hit.orig_file_path.name.lower()
    if name not in (".htaccess", ".user.ini") and not re.search(
        r"\.(?:php[3-8]?|phtml|inc|js|html?|cgi|pl|sh|py)$", name
    ):
        return False
    return True


def _system_owned_copy_count(file_hash):
    # Server-wide, not just this owner or cleanup batch. Stop at the cutoff.
    _register_system_owned_text(MalwareHit._meta.database)
    owner = MalwareHit.owner
    uid = owner.cast("INTEGER")
    # A round-trip excludes SQLite's partial parses and saturated overflows.
    # Canonical integers stay in SQL; other int()-accepted forms retain the
    # exact existing predicate through the registered SQLite function.
    system_owner = Case(
        None,
        (
            (
                owner == uid.cast("TEXT"),
                (uid < utils.get_min_uid()) & uid.not_in(WEB_EXEC_UIDS),
            ),
        ),
        fn.imav_system_owned_text(owner),
    )
    copies = (
        MalwareHit.select(MalwareHit.orig_file)
        .where(
            MalwareHit.hash == file_hash,
            MalwareHit.resource_type == MalwareScanResourceType.FILE.value,
            system_owner,
        )
        .distinct()
        .limit(10)
    )
    return len(list(copies.tuples().iterator()))


async def _root_cleanup_copy_counts(hits, capable, *, curator=None):
    if not capable or not feature_flags.is_enabled(
        "clean_system_owned_site_files"
    ):
        return {}
    if curator is None:
        curator = await curator_version.selected_curator()
    hashes = {
        hit.hash
        for hit in hits
        if is_uid(hit.owner)
        and _system_owned(int(hit.owner))
        and not is_crontab(hit.orig_file_path)
        and _root_cleanup_candidate(hit, capable, curator=curator)
    }
    if not hashes:
        return {}
    # One worker-owned DB step per batch, one bounded query per distinct hash.
    return await store_db.write(
        lambda: {value: _system_owned_copy_count(value) for value in hashes}
    )


def _root_cleanup_in_scope(
    hit, capable=False, *, copy_counts=None, curator=None
) -> bool:
    """Agent scope gate; async callers supply executor-computed counts."""
    if not _root_cleanup_candidate(hit, capable, curator=curator):
        return False
    count = (
        _system_owned_copy_count(hit.hash)
        if copy_counts is None
        else copy_counts.get(hit.hash, 10)
    )
    return count < 10


def _refused_by_owner_guard(
    hit, capable=False, *, copy_counts=None, curator=None
) -> bool:
    return (
        is_uid(hit.owner)
        and _system_owned(int(hit.owner))
        and not is_crontab(hit.orig_file_path)
        and not _root_cleanup_in_scope(
            hit, capable, copy_counts=copy_counts, curator=curator
        )
    )


_group_by_status = partial(MalwareHit.group_by_attribute, attribute="status")
_group_by_user = partial(MalwareHit.group_by_attribute, attribute="owner")


def _terminal_refusal_attempts(paths, since):
    return list(
        MalwareHistory.get_terminal_refusal_attempts(paths, since=since)
    )


def _still_owned(paths, uid):
    """The *paths* still regular files of *uid*. Runs in the executor."""
    return {path for path in paths if run_as.still_owned(path, uid)}


_PLANNED_REFUSAL = "_unnamed_owner_refusal"


def _planned_refusal(hit):
    """The refusal reason the plan put on *hit*, else None. Only the hit's
    own attributes count: an object that answers for any attribute (a test
    double, a proxy) has not been planned for."""
    return getattr(hit, "__dict__", {}).get(_PLANNED_REFUSAL)


async def _unnamed_owner_refusals(uid: int, hits):
    """For *hits* of an owner *uid* without a user name: the (uid, gid) to
    clean them as, or None, and the hits refused, by reason. A home that
    the panel is removing is never cleaned; the other hits are refused
    unless they can be cleaned as their owner UID."""
    refused = defaultdict(list)
    rest = []
    for hit in hits:
        if run_as.removal_in_progress(hit.orig_file):
            refused[run_as.ACCOUNT_REMOVAL].append(hit)
        else:
            rest.append(hit)
    identity = await run_as.identity(uid) if rest else None
    if rest and identity is None:
        refused[run_as.OWNER_UNRESOLVED].extend(rest)
    return identity, refused


def _homes(users):
    homes = {}
    for user in users:
        try:
            homes[user] = pwd.getpwnam(user).pw_dir
        except Exception as error:
            logger.debug("Resolving cleanup account home failed: %r", error)
            homes[user] = ""
    return homes


def _kernel() -> str:
    return os.uname().release


throttled_log_error = rate_limit(period=DAY, on_drop=logger.warning)(
    logger.error
)


def _not_recorded(step, who, count, put_back, error):
    """One ERROR (it reaches Sentry) for a database step whose lock retries
    ran out: *count* files have their state or history not recorded,
    *put_back* files wait in the proxy for the next cleanup run."""
    logger.error(
        "Cleanup %s for %s: database remained locked after 3 attempts;"
        " %d files not recorded, %d put back for the next cleanup run: %s",
        step,
        who,
        count,
        put_back,
        error,
    )


def _owners(hits) -> str:
    return ", ".join(sorted({str(hit.owner) for hit in hits}))


def _databases(hits) -> str:
    return ", ".join(
        sorted({"{} ({})".format(hit.orig_file, hit.app_name) for hit in hits})
    )


def _claim(hits):
    """Mark *hits* cleanup started and return them. The rows of one file are
    one file (older agents could store two): the found rows of a file in
    this claim are claimed together; a found row whose file another row has
    under cleanup, or cleaned after this detection, is that detection
    handled already and is retired, as the scan store retires a replaced
    row. A hit whose stored status is no longer the one it was read with, or
    whose row is gone while another row tracks the file, belongs to another
    cleanup and is left alone. A backup per path replaced by a second start
    would hold the cleaned file instead of the original."""
    with MalwareHit._meta.database.atomic():
        # one pass over the rows: the claim holds the writer lock, so a row
        # is not compared with every other row of its file
        stored, count = {}, defaultdict(int)
        started, cleaned = defaultdict(set), defaultdict(list)
        for (
            row_id,
            path,
            status,
            cleaned_at,
            detected,
        ) in get_results_iterable_expression(
            lambda paths: MalwareHit.select(
                MalwareHit.id,
                MalwareHit.orig_file,
                MalwareHit.status,
                MalwareHit.cleaned_at,
                MalwareHit.timestamp,
            )
            .where(MalwareHit.orig_file.in_(paths))
            .tuples(),
            sorted({hit.orig_file for hit in hits}),
        ):
            stored[path, row_id] = status, detected
            count[path] += 1
            if status == MalwareHitStatus.CLEANUP_STARTED:
                started[path].add(row_id)
            elif status in MalwareHitStatus.CLEANED and cleaned_at is not None:
                cleaned[path].append((cleaned_at, row_id))
        # the two latest cleans: the latest one of another row
        latest = {
            path: heapq.nlargest(2, rows) for path, rows in cleaned.items()
        }

        def handled(hit):
            path = hit.orig_file
            return len(started[path]) > (hit.id in started[path]) or (
                hit.timestamp is not None
                and any(
                    cleaned_at > hit.timestamp
                    for cleaned_at, row_id in latest.get(path, ())
                    if row_id != hit.id
                )
            )

        claimed, retired = [], []
        for hit in hits:
            # its own row: the id, path and detection time (a newer
            # detection of the file may have been given the id)
            row = stored.get((hit.orig_file, hit.id))
            own = row is not None and row[1] == hit.timestamp
            others = count[hit.orig_file] - own
            if (own and row[0] != hit.status) or (not own and others):
                continue
            if handled(hit):
                if own:
                    retired.append(hit)
                continue
            claimed.append(hit)
        if retired:
            execute_iterable_expression(
                lambda ids: MalwareHit.delete().where(
                    MalwareHit.id.in_(ids),
                    MalwareHit.status == MalwareHitStatus.FOUND,
                ),
                [hit.id for hit in retired],
            )
            logger.info(
                "Retired %d found hit(s) whose file another cleanup has"
                " under way or cleaned since their detection",
                len(retired),
            )
        # a hit whose row is gone may have lent its id to a new row
        MalwareHit.set_status(
            claimed, MalwareHitStatus.CLEANUP_STARTED, same_detection=True
        )
    return claimed


def _database_finding(hit):
    """What a database hit's row holds: its database and the scan. As
    stored text: a hit made in memory may carry the port as a number."""
    return tuple(
        None if value is None else str(value)
        for value in (
            hit.orig_file,
            hit.app_name,
            hit.db_host,
            hit.db_port,
            hit.db_name,
            hit.scanid_id,
        )
    )


def _share_outcome(twins, first, succeeded, failed, not_exist):
    """The backup outcome of a file for the other rows of the file."""
    ok = {id(hit) for hit in succeeded}
    gone = {id(hit) for hit in not_exist}
    failed = dict(failed)
    for twin in twins:
        if first[twin.orig_file] in failed:
            failed[twin] = failed[first[twin.orig_file]]
    return (
        [
            *succeeded,
            *(t for t in twins if id(first[t.orig_file]) in ok),
        ],
        failed,
        [
            *not_exist,
            *(t for t in twins if id(first[t.orig_file]) in gone),
        ],
    )


async def _with_history(action, hits, **kwargs):
    """Run a MalwareAction once; only its history write is retried on a
    lock, in a worker connection (the scan store's history_writer)."""
    token = store_db.history_writer.set(store_db.write)
    try:
        return await action(hits, **kwargs)
    finally:
        store_db.history_writer.reset(token)


def filter_cleanable(hits: Iterable[MalwareHit]) -> Iterable:
    return (hit for hit in hits if hit.status == MalwareHitStatus.FOUND)


def drop_stored_originals(hits: list) -> list:
    """Forget hits that point inside the cleanup storage.

    They are copies of files detected elsewhere, and they can never be
    cleaned: only root may enter the storage, while the cleanup drops
    privileges to the owner of the file the copy was made from.
    """
    stored = [hit for hit in hits if CleanupStorage.contains(hit.orig_file)]
    if not stored:
        return hits
    logger.warning(
        "Removing %s hit(s) inside %s: a stored original is not a detection"
        " of its own",
        len(stored),
        CleanupStorage.path,
    )
    MalwareHit.delete_instances(stored)
    stored_ids = {hit.id for hit in stored}
    return [hit for hit in hits if hit.id not in stored_ids]


def sweep_interrupted_runs():
    try:
        sweep_orphaned_runs()
    except Exception:
        logger.exception("Could not remove files of interrupted cleanups")


def release_frozen_orphans():
    """SIGCONT every process the previous agent left frozen (its registry
    entry, pid and start time still matching); the entries released."""
    try:
        released = release_orphans()
    except Exception:
        logger.exception("Could not release processes frozen earlier")
        return []
    for fp in released:
        logger.info(
            "Released pid %s, left frozen for %s by an earlier stop step",
            fp.pid,
            fp.file,
        )
    return released


class _StopCleanupCancelled(asyncio.CancelledError):
    def __init__(self, state):
        super().__init__()
        self.state = state


class Cleanup(MessageSink, MessageSource):
    def __init__(self):
        self._cleanup_task = None
        self._store_original_task = None
        self._running = False
        self._loop = None
        self._sink = None
        self._proxy = None
        self._cleaner = None
        self._stop_controller = None
        self._retry_task = None
        self._gone_pending = []  # (hit, cause, initiator) of gone files
        self._gone_attempts = {}  # id(entry) -> (owned entry, attempts)
        self._cron_checked_at = float("-inf")
        self._runs_swept_at = None  # the agent start sweeps them first
        self._run_reports = []  # (out, hits, cause, initiator) of one run
        # (hits, cause, initiator, post_action, scan_id, standard_only) a
        # run could not record, for the proxy once the run is over
        self._put_back = []
        self._rollup_loaded = False

    async def create_source(self, loop, sink):
        self._loop = loop
        self._sink = sink
        # Load before constructors as well as before the first await.  If
        # startup fails, shutdown may save only state it actually owns.
        stop_report.load_rollup()
        self._rollup_loaded = True
        self._proxy = MalwareCleanupProxy()
        _, sweep_cancelled = await stop_report._complete_owned(
            loop.run_in_executor(None, blacklist_hashes.remove_orphans)
        )
        _, sweep_cancelled = await stop_report._complete_owned(
            loop.run_in_executor(None, sa_verdicts.remove_orphans),
            cancelled=sweep_cancelled,
        )
        self._cleaner = MalwareCleaner(loop=loop, sink=sink)
        # the files of a run the previous agent did not see end
        _, sweep_cancelled = await stop_report._complete_owned(
            loop.run_in_executor(None, sweep_interrupted_runs),
            cancelled=sweep_cancelled,
        )
        if self._stop_controller is None:
            self._stop_controller = StopProcessController()
        # a process frozen when the agent died must not stay frozen
        released, release_cancelled = await stop_report._complete_owned(
            loop.run_in_executor(None, release_frozen_orphans),
            cancelled=sweep_cancelled,
        )
        report_cancelled = False
        try:
            await stop_report.report_released(
                self._sink,
                released,
                kernel=_kernel(),
                panel=HostingPanel().NAME,
            )
        except asyncio.CancelledError:
            report_cancelled = True
        except Exception:
            logger.exception("Could not report the processes released")
        if release_cancelled or report_cancelled:
            raise asyncio.CancelledError
        self._cleanup_task = loop.create_task(self.cleanup())

    async def create_sink(self, loop):
        pass

    async def shutdown(self):
        if self._cleanup_task:
            await safe_cancel_task(self._cleanup_task)
        try:
            await self._stop_retry_task()
        finally:
            if stop_registry.REGISTRY_PATH.exists():
                # no process may stay frozen once the agent is gone
                await self._loop.run_in_executor(None, release_frozen_orphans)

    async def _stop_retry_task(self):
        if self._retry_task is not None and not self._retry_task.done():
            task = self._retry_task
            task.cancel()
            caller_cancelled = False
            while not task.done():
                try:
                    await asyncio.shield(task)
                except asyncio.CancelledError:
                    current = asyncio.current_task()
                    caller_cancelled = caller_cancelled or bool(
                        current and current.cancelling()
                    )
            try:
                task.result()
            except asyncio.CancelledError:
                pass
            if caller_cancelled:
                raise asyncio.CancelledError
        # the roll-up is not sent at exit (the agent exits when idle):
        # the next agent sends it when its hour is over
        if self._rollup_loaded:
            stop_report.save_rollup()

    @expect(MessageType.MalwareCleanupTask)
    async def process_cleanup_task(self, message: Dict):
        cause = message.get("cause")
        initiator = message.get("initiator")
        post_action = message.get("post_action")
        scan_id = message.get("scan_id")
        standard_only = message.get("standard_only")

        manual_cleanup = cause is None
        # In case another scan already found some of the hits
        # and the cleanup for them has started.
        origin_hits_num = len(message["hits"])
        hits = MalwareHit.refresh_hits(
            message["hits"], include_scan_info=not manual_cleanup
        )
        hits = filter_cleanable(hits)
        hits = [
            hit
            for hit in hits
            if hit.resource_type == MalwareScanResourceType.FILE.value
        ]
        hits = drop_stored_originals(hits)
        hits = await self._mark_unnamed_owner_refusals(
            hits, drop_recorded=not manual_cleanup
        )
        if (
            not manual_cleanup
        ):  # don't use any limits when run cleanup manually
            rescan_hits, hits = self._split_hits_by_scan_type(
                hits, [MalwareScanType.RESCAN, MalwareScanType.RESCAN_OUTDATED]
            )
            rescan_hits = await self._filter_rescan_hits(rescan_hits)
            hits = rescan_hits + await self._filter_failed_to_cleanup_hits(
                hits
            )
        if filtered := origin_hits_num - len(hits):
            logger.info(
                "%s/%s hits filtered before cleanup",
                filtered,
                origin_hits_num,
            )
        self._store_original_task = self._loop.create_task(
            self._store_original(
                hits, cause, initiator, post_action, scan_id, standard_only
            )
        )

    @staticmethod
    async def _mark_unnamed_owner_refusals(hits, *, drop_recorded):
        """Mark the hits that _clean_files is going to refuse because their
        owner UID has no user name, so that no backup copy is made of them.
        With *drop_recorded* (an automatic cleanup), leave out the ones
        whose refusal is on record already: it cannot change while the
        owner and path stay the same, and it is reported once per file."""
        candidates = defaultdict(list)
        for hit in hits:
            if is_uid(hit.owner) and not _system_owned(int(hit.owner)):
                candidates[int(hit.owner)].append(hit)
        if not candidates or not LicenseCLN.is_unlimited():
            return hits
        marked = []
        for uid, owner_hits in candidates.items():
            if await get_username_by_uid(uid):
                continue
            _, refused = await _unnamed_owner_refusals(uid, owner_hits)
            for reason, refused_hits in refused.items():
                for hit in refused_hits:
                    setattr(hit, _PLANNED_REFUSAL, reason)
                    marked.append(hit)
        if not drop_recorded or not marked:
            return hits
        recorded = set()
        for chunk in split_for_chunk(marked, chunk_size=200):
            recorded.update(
                await store_db.write(
                    MalwareHistory.get_terminal_refusals,
                    [hit.orig_file for hit in chunk],
                )
            )
        dropped = {
            id(hit)
            for hit in marked
            if (hit.orig_file, hit.owner, _planned_refusal(hit)) in recorded
        }
        if dropped:
            logger.info(
                "%s hit(s) not cleaned: refused before, their owner UID has"
                " no user name",
                len(dropped),
            )
        return [hit for hit in hits if id(hit) not in dropped]

    @staticmethod
    def _split_hits_by_scan_type(
        hits: list, scan_types: List[MalwareScanType]
    ) -> Tuple[list, list]:
        target_hits, other_hits = [], []
        for hit in hits:
            if hit.scanid.type in scan_types:
                target_hits.append(hit)
            else:
                other_hits.append(hit)
        return target_hits, other_hits

    async def _filter_failed_to_cleanup(
        self,
        hits: list,
        *,
        time_range: float,
        allowed_total: int,
        allowed_after_success: Optional[int] = None,
        report_give_up: bool = False,
    ) -> list:
        hits_to_clean = []
        suppressed = []
        if hits:
            since = time.time() - time_range
            attempts = {}
            for hits_chunk in split_for_chunk(hits, chunk_size=200):
                paths = [hit.orig_file for hit in hits_chunk]
                counted = MalwareHistory.get_failed_cleanup_attempts(
                    paths, since=since
                )
                for path, total, after_success in counted:
                    attempts[path] = (total, after_success)
                # a refusal that no retry can change spends no attempt
                for path, total, after_success in await store_db.write(
                    _terminal_refusal_attempts, paths, since
                ):
                    spent, spent_after = attempts.get(path, (0, 0))
                    attempts[path] = (
                        spent - total,
                        spent_after - (after_success or 0),
                    )
                await asyncio.sleep(0)
            for hit in hits:
                total, after_success = attempts.get(hit.orig_file, (0, 0))
                exhausted = total >= allowed_total or (
                    allowed_after_success is not None
                    and after_success >= allowed_after_success
                )
                if exhausted:
                    throttled_log_error(
                        "Skip cleanup file '%s', since there are too many "
                        "attempts to cleanup it in %s sec [%s, %s since the "
                        "last successful cleanup]",
                        hit.orig_file,
                        time_range,
                        total,
                        after_success,
                    )
                    if hit.orig_file not in suppressed:
                        suppressed.append(hit.orig_file)
                    continue
                hits_to_clean.append(hit)
        if suppressed and report_give_up:
            await self._report_suppressed_cleanup(suppressed, time_range)
        return hits_to_clean

    async def _report_suppressed_cleanup(self, paths: list, time_range: float):
        # The only signal that the agent decided not to try: without it a
        # path that is being skipped looks exactly like one with nothing
        # left to do.
        await self._sink.process_message(
            MessageType.CleanupFailed(
                message=(
                    "Automatic cleanup suppressed for {} path(s) that"
                    " exhausted the attempt budget in {} sec: {}".format(
                        len(paths), time_range, ", ".join(paths[:3])
                    )
                ),
                exception="CleanupAttemptsExhausted",
                err="",
                timestamp=int(time.time()),
            )
        )

    async def _filter_rescan_hits(self, hits: list) -> list:
        # A burst guard, not a give-up decision: a success minutes ago is no
        # reason to keep hammering the same path, so no reset applies here.
        return await self._filter_failed_to_cleanup(
            hits, time_range=5 * MINUTE, allowed_total=RESCAN_ATTEMPTS
        )

    async def _filter_failed_to_cleanup_hits(self, hits: list) -> list:
        return await self._filter_failed_to_cleanup(
            hits,
            time_range=DAY,
            report_give_up=True,
            allowed_total=(
                MalwareTune.CLEANUP_FAILURES_PER_DAY
                or COUNT_OF_FAILURES_TO_CLEANUP_PER_DAY
            ),
            allowed_after_success=(
                MalwareTune.CLEANUP_ATTEMPTS_AFTER_SUCCESS
                or COUNT_OF_ATTEMPTS_TO_CLEANUP_PER_DAY
            ),
        )

    async def _store_original(
        self, hits, cause, initiator, post_action, scan_id, standard_only
    ):
        # A previous attempt's backup must not authorize this queued batch.
        # This private attribute follows the hit through proxy/deferred queues;
        # it is not a model field and is never serialized or persisted.
        for hit in hits:
            if hit.owner == "0":
                hit._cleanup_backup_stored = False
        curator = await curator_version.selected_curator()
        capable = await curator_version.system_owned_cleanup_available(curator)
        # Admission may exhaust the database retry budget. Do it before the
        # claim so that an unbacked batch remains found and retryable.
        try:
            copy_counts = await _root_cleanup_copy_counts(
                hits, capable, curator=curator
            )
        except store_db.StoreBusyError as exc:
            _not_recorded(
                "root-copy admission", _owners(hits), len(hits), 0, exc
            )
            return
        try:
            claimed = await store_db.write(_claim, hits)
        except store_db.StoreBusyError as exc:
            # nothing is marked: the hits stay found, as before the task
            _not_recorded("start", _owners(hits), len(hits), 0, exc)
            return
        if taken := len(hits) - len(claimed):
            logger.info(
                "%s/%s hits taken or handled by another cleanup before this"
                " one started",
                taken,
                len(hits),
            )
        hits = claimed
        original_status = _group_by_status(hits)
        # Re-check ignore list after marking as started, before any destructive action
        hits_to_keep = []
        ignored_hits = []
        for hit in hits:
            try:
                if await MalwareIgnorePath.is_path_ignored(hit.orig_file):
                    ignored_hits.append(hit)
                else:
                    hits_to_keep.append(hit)
            except Exception as exc:
                # Be conservative: if ignore check fails, keep the hit
                logger.exception(
                    "Ignore re-check failed for file %s: %s; keeping hit",
                    hit.orig_file,
                    exc,
                )
                hits_to_keep.append(hit)
        if ignored_hits:
            # Transition ignored hits to a terminal non-in-progress state
            try:
                await store_db.write(
                    MalwareHit.set_status,
                    ignored_hits,
                    MalwareHitStatus.FOUND,
                    same_detection=True,
                )
            except store_db.StoreBusyError as exc:
                # left cleanup_started until the agent starts again
                _not_recorded(
                    "ignored-file reset",
                    _owners(ignored_hits),
                    len(ignored_hits),
                    0,
                    exc,
                )
            # Do not proceed with storage/cleanup for ignored hits
            hits = hits_to_keep
        # _clean_files refuses these before the cleaner ever runs, so a
        # backup copy of them is work nobody will read.
        refused = [
            hit
            for hit in hits
            if _refused_by_owner_guard(
                hit, capable, copy_counts=copy_counts, curator=curator
            )
            or _planned_refusal(hit)
        ]
        if refused:
            refused_set = set(refused)
            hits = [hit for hit in hits if hit not in refused_set]
        # one backup per file: the rows of a file share it and its outcome
        first = {}
        for hit in hits:
            first.setdefault(hit.orig_file, hit)
        twins = [hit for hit in hits if first[hit.orig_file] is not hit]
        # a failed backup writes its history inside the storage loop: retried
        # on a lock like the others, and one that stays locked ends neither
        # the loop nor this start
        unrecorded = []

        async def failed_history(callback):
            try:
                return await store_db.write(callback)
            except store_db.StoreBusyError as exc:
                unrecorded.append(exc)

        token = store_db.history_writer.set(failed_history)
        try:
            with inactivity.track.task("cleanup_storage"):
                succeeded, failed, not_exist = await CleanupStorage.store_all(
                    [*first.values()] if twins else hits
                )
        finally:
            store_db.history_writer.reset(token)
        if unrecorded:
            _not_recorded(
                "failed-backup history",
                _owners(failed),
                len(unrecorded),
                0,
                unrecorded[-1],
            )
        for hit, exc in failed.items():
            await self._sink.process_message(
                MessageType.CleanupFailed(
                    message=(
                        "Failed to store the original from {} to {}".format(
                            hit.orig_file, CleanupStorage.path
                        )
                    ),
                    exception=type(exc).__name__,
                    err=str(exc),
                    timestamp=int(time.time()),
                )
            )

        if twins:
            succeeded, failed, not_exist = _share_outcome(
                twins, first, succeeded, failed, not_exist
            )
        succeeded = list(succeeded)
        for hit in succeeded:
            if hit.owner == "0":
                hit._cleanup_backup_stored = True
        self._add_to_proxy(
            [*succeeded, *refused],
            cause,
            initiator,
            post_action,
            scan_id,
            standard_only,
        )

        failed_ids = {hit.id for hit in failed}
        resets = [
            (status, [hit for hit in hit_list if hit.id in failed_ids])
            for status, hit_list in original_status.items()
        ]
        for index, (status, hit_list) in enumerate(resets):
            try:
                await store_db.write(
                    MalwareHit.set_status,
                    hit_list,
                    status,
                    same_detection=True,
                )
            except store_db.StoreBusyError as exc:
                # left cleanup_started until the agent starts again
                left = [hit for _, rest in resets[index:] for hit in rest]
                _not_recorded(
                    "failed-backup reset", _owners(left), len(left), 0, exc
                )
                break
        # Own missing-file evidence before deleting its database row or
        # awaiting history I/O.  Either operation can fail or be cancelled,
        # and the recorded inode is then the only input left for a retry.
        self._gone_pending.extend(
            (hit, cause, initiator)
            for hit in not_exist
            if hit.resource_type == MalwareScanResourceType.FILE.value
        )
        step = "gone-file delete"
        try:
            await store_db.write(
                MalwareHit.delete_instances, not_exist, same_detection=True
            )
            step = "gone-file history"
            await _with_history(
                MalwareAction.not_exist,
                not_exist,
                cause=cause,
                initiator=initiator,
            )
        except store_db.StoreBusyError as exc:
            # the claimed entries above keep the stop step for these files
            _not_recorded(step, _owners(not_exist), len(not_exist), 0, exc)

    def _add_to_proxy(
        self, hits, cause, initiator, post_action, scan_id, standard_only
    ):
        standard_only_hits = []
        advanced_hits = []
        for hit in hits:
            standard_only_user = decide_if_standard_signatures_only(
                initiator, standard_only
            )
            if standard_only_user:
                standard_only_hits.append(hit)
            else:
                advanced_hits.append(hit)

        self._proxy.add(
            cause,
            initiator,
            post_action,
            scan_id,
            True,
            standard_only_hits,
        )
        self._proxy.add(
            cause,
            initiator,
            post_action,
            scan_id,
            standard_only,  # None if default action otherwise False
            advanced_hits,
        )

    @staticmethod
    def _user_hits(hits):
        user_hits = _group_by_user(hits)
        return user_hits

    def _cloud_assisted_hits(self):
        action_hits = self._proxy.flush()

        for (
            cause,
            initiator,
            post_action,
            scan_id,
            standard_only,
            all_hits,
        ) in action_hits:
            blacklist = [
                hit
                for hit in all_hits
                if re.match(r"\w+-BLKH-|cloudhash\.|cld-", hit.type)
            ]
            regular_hits = [hit for hit in all_hits if hit not in blacklist]
            yield (
                regular_hits,
                blacklist,
                cause,
                initiator,
                post_action,
                scan_id,
                standard_only,
            )

    async def _start_hook(self, cleanup_id, started, hits):
        dump = [hit.as_dict() for hit in hits]
        cleanup_started = HookEvent.MalwareCleanupStarted(
            cleanup_id=cleanup_id,
            started=started,
            total_files=len(hits),
            DUMP=dump,
        )
        await self._sink.process_message(cleanup_started)

    async def _clean_files(
        self,
        hits,
        blacklist=None,
        cause=None,
        initiator=None,
        post_action=None,
        scan_id=None,
        standard_only=None,
        gone_entries=None,
    ):
        curator = await curator_version.selected_curator()
        capable = await curator_version.system_owned_cleanup_available(curator)
        user_hits = self._user_hits(hits)
        user_hits_black = self._user_hits(blacklist or [])
        gone_entries = gone_entries if gone_entries is not None else []
        batch = [*hits, *(blacklist or [])]
        scope = (cause, initiator, post_action, scan_id, standard_only)
        try:
            copy_counts = await _root_cleanup_copy_counts(
                [*batch, *(entry[0] for entry in gone_entries)],
                capable,
                curator=curator,
            )
        except store_db.StoreBusyError as exc:
            # Backups already exist. Keep every claimed hit in the existing
            # deferred queue; no file or process is touched in this run.
            self._put_back_batch(
                batch,
                [],
                "root-copy admission",
                _owners(batch),
                len(batch),
                exc,
                *scope,
            )
            return
        users = sorted({*user_hits, *user_hits_black})

        def unvisited_hits(after_index):
            remaining = []
            for unvisited_user in users[after_index + 1 :]:
                remaining.extend(user_hits.get(unvisited_user, []))
                remaining.extend(user_hits_black.get(unvisited_user, []))
            return remaining

        def requeue(pending):
            if pending:
                self._add_to_proxy(
                    pending,
                    cause,
                    initiator,
                    post_action,
                    scan_id,
                    standard_only,
                )

        for user_index, user in enumerate(users):
            hits_regular = user_hits.get(user, [])
            hits_black = user_hits_black.get(user, [])
            batch = hits_regular + hits_black
            reset = []  # ignored hits found again: never put back

            # Filter out ignored paths before invoking cleaner (DEF-36692)
            try:
                hits_regular = await self._drop_ignored(hits_regular, reset)
                hits_black = await self._drop_ignored(hits_black, reset)
            except store_db.StoreBusyError as exc:
                self._put_back_batch(
                    batch, reset, "ignored-file reset", user, 1, exc, *scope
                )
                continue
            user_hits_all = hits_regular + hits_black

            # DEF-40165: skip files that no longer exist on disk so they
            # are not dispatched to procu2 (guaranteed to fail and loop).
            # exists_async() checks concurrently in k8s without blocking
            # the event loop.
            all_hits = hits_regular + hits_black
            existence = await gather_exists(
                [h.orig_file_path for h in all_hits]
            )
            preflight_not_exist = [
                h for h, exists in zip(all_hits, existence) if not exists
            ]
            # a deleted file may still be running or held: the stop step
            # looks for its processes by the recorded inode (the cleaner
            # is never asked about it); the owner's files already gone at
            # the storage step join here and leave the pending list
            gone = [
                h
                for h in preflight_not_exist
                if h.resource_type == MalwareScanResourceType.FILE.value
            ]
            mine = [e for e in gone_entries if e[0].owner == user]
            for e in mine:
                gone_entries[:] = [
                    pending for pending in gone_entries if pending is not e
                ]
                gone.append(e[0])
            new_gone_entries = [
                (hit, cause, initiator)
                for hit in gone
                if all(entry[0] is not hit for entry in mine)
            ]
            # Claim newly vanished hits before deleting their DB rows or
            # awaiting history writes.  Cancellation/error can then retry
            # the exact in-memory evidence on a later cleanup tick.
            self._gone_pending.extend(new_gone_entries)
            owned_gone_entries = mine + new_gone_entries
            if preflight_not_exist:
                preflight_set = set(preflight_not_exist)
                hits_regular = [
                    h for h in hits_regular if h not in preflight_set
                ]
                hits_black = [h for h in hits_black if h not in preflight_set]
                step = "gone-file delete"
                # a put-back hit whose row this cleanup deleted gets only its
                # history: SQLite may have given the id to a new row since
                deleted = [
                    hit
                    for hit in preflight_not_exist
                    if not getattr(hit, "_row_deleted", False)
                ]
                try:
                    if deleted:
                        await store_db.write(
                            MalwareHit.delete_instances,
                            deleted,
                            same_detection=True,
                        )
                        for hit in deleted:
                            hit._row_deleted = True
                    step = "gone-file history"
                    await _with_history(
                        MalwareAction.not_exist,
                        preflight_not_exist,
                        cause=cause,
                        initiator=initiator,
                    )
                except store_db.StoreBusyError as exc:
                    # The next run finds a put-back gone file and claims it
                    # again: this run's claim goes, the hit carries the same
                    # evidence. Claims of earlier runs stay pending.
                    self._ack_gone_entries(new_gone_entries)
                    self._put_back_batch(
                        batch,
                        reset,
                        step,
                        user,
                        len(preflight_not_exist),
                        exc,
                        *scope,
                    )
                    continue
                user_hits_all = [
                    h for h in user_hits_all if h not in preflight_set
                ]

            # Skip if nothing to clean after filtering
            if not hits_regular and not hits_black:
                if gone:
                    try:
                        await self._stop_running_processes(
                            user,
                            [],
                            [],
                            standard_only,
                            cause,
                            initiator,
                            gone,
                            gone_entries=owned_gone_entries,
                            **(
                                {
                                    "copy_counts": copy_counts,
                                    "curator": curator,
                                }
                                if curator
                                else {}
                            ),
                        )
                    except _StopCleanupCancelled:
                        requeue(unvisited_hits(user_index))
                        raise asyncio.CancelledError from None
                continue

            # the cleaner wrapper needs bare in-container paths; the
            # tenant rides in --username (k8s prefix stripped)
            # once per file: the rows of a file share its result
            files = list(
                dict.fromkeys(str(hit.orig_file_path) for hit in hits_regular)
            )
            black = list(
                dict.fromkeys(str(hit.orig_file_path) for hit in hits_black)
            )

            logger.debug("Cleaning files: %s", files + black)
            cleanup_id = uuid.uuid4().hex
            started = time.time()
            identity = None
            if is_uid(user):  # non panel user
                uid = int(user)

                async def _refuse(error, refused_hits, reason=None):
                    await self._sink.process_message(
                        MessageType.MalwareCleanup(
                            hits=refused_hits,
                            result={},
                            cleanup_id=cleanup_id,
                            started=started,
                            error=error,
                            cause=cause,
                            initiator=initiator,
                            post_action=post_action,
                            scan_id=scan_id,
                            args=[],
                            **({"refusal_reason": reason} if reason else {}),
                        )
                    )

                if _system_owned(uid):
                    # A relaxed flag/count at dispatch cannot upgrade a
                    # backup-stage refusal. Row IDs may be reused after a
                    # gone hit is deleted: authorize each live object from its
                    # own backup decision. Gone hits only feed the stop step.
                    allowed_objects = {
                        id(hit)
                        for hit in user_hits_all
                        if not _refused_by_owner_guard(
                            hit,
                            capable,
                            copy_counts=copy_counts,
                            curator=curator,
                        )
                        and (
                            hit.owner != "0"
                            or getattr(hit, "_cleanup_backup_stored", False)
                        )
                    }
                    refused = [
                        hit
                        for hit in user_hits_all
                        if id(hit) not in allowed_objects
                    ]
                    if refused:
                        logger.warning(
                            "Refusing to clean files owned by system"
                            " account %s, manual cleanup required",
                            uid,
                        )
                        await _refuse(
                            "Cleanup skipped. File is owned by a system"
                            " account, manual cleanup required",
                            refused,
                        )
                    if not allowed_objects and not any(
                        not _refused_by_owner_guard(
                            hit,
                            capable,
                            copy_counts=copy_counts,
                            curator=curator,
                        )
                        for hit in gone
                    ):
                        self._ack_gone_entries(owned_gone_entries)
                        continue
                    hits_regular = [
                        h for h in hits_regular if id(h) in allowed_objects
                    ]
                    hits_black = [
                        h for h in hits_black if id(h) in allowed_objects
                    ]
                    user_hits_all = hits_regular + hits_black
                    files = list(
                        dict.fromkeys(
                            str(hit.orig_file_path) for hit in hits_regular
                        )
                    )
                    black = list(
                        dict.fromkeys(
                            str(hit.orig_file_path) for hit in hits_black
                        )
                    )
                    if not files and not black:
                        if gone:
                            try:
                                await self._stop_running_processes(
                                    user,
                                    [],
                                    [],
                                    standard_only,
                                    cause,
                                    initiator,
                                    gone,
                                    gone_entries=owned_gone_entries,
                                    **(
                                        {
                                            "copy_counts": copy_counts,
                                            "curator": curator,
                                        }
                                        if curator
                                        else {}
                                    ),
                                )
                            except _StopCleanupCancelled:
                                requeue(unvisited_hits(user_index))
                                raise asyncio.CancelledError from None
                        continue
                    logger.warning(
                        "Cleaning in-scope system-owned files as a deliberate"
                        " guard exception: %s",
                        files + black,
                    )
                if not LicenseCLN.is_unlimited():
                    logger.error(
                        f"Can't clean files for non panel user {uid=}, "
                        "since license is limited"
                    )
                    await _refuse(
                        "Cleanup failed. Automatic cleanup of files not"
                        " owned by a panel user requires an unlimited"
                        " license",
                        user_hits_all,
                    )
                    self._ack_gone_entries(owned_gone_entries)
                    continue
                # the refusals planned before the backup step hold, whatever
                # the lookup below answers: a hit refused there has no
                # backup copy to restore from. The next round retries it.
                refused = defaultdict(list)
                planned = []
                for hit in user_hits_all:
                    if reason := _planned_refusal(hit):
                        refused[reason].append(hit)
                    else:
                        planned.append(hit)
                if username := await get_username_by_uid(uid):
                    user = username
                else:
                    logger.warning(f"Can't find username for {uid=}")
                    identity, more = await _unnamed_owner_refusals(
                        uid, planned
                    )
                    for reason, refused_hits in more.items():
                        refused[reason].extend(refused_hits)
                    if identity is not None:
                        refused_ids = {
                            id(hit)
                            for refused_hits in refused.values()
                            for hit in refused_hits
                        }
                        to_check = [
                            hit
                            for hit in planned
                            if id(hit) not in refused_ids
                        ]
                        # the owner right before the run (an lstat of a
                        # local file: cluster mode never cleans as a UID)
                        owned = await self._loop.run_in_executor(
                            None,
                            _still_owned,
                            [str(hit.orig_file_path) for hit in to_check],
                            uid,
                        )
                        if mismatched := [
                            hit
                            for hit in to_check
                            if str(hit.orig_file_path) not in owned
                        ]:
                            refused[run_as.OWNER_MISMATCH].extend(mismatched)
                refused_ids = set()
                for reason, refused_hits in refused.items():
                    logger.warning(
                        "Refusing to clean %d file(s) of uid=%d: %s",
                        len(refused_hits),
                        uid,
                        reason,
                    )
                    await _refuse(run_as.ERRORS[reason], refused_hits, reason)
                    refused_ids.update(id(h) for h in refused_hits)
                if refused_ids:
                    hits_regular = [
                        h for h in hits_regular if id(h) not in refused_ids
                    ]
                    hits_black = [
                        h for h in hits_black if id(h) not in refused_ids
                    ]
                    user_hits_all = hits_regular + hits_black
                    if not user_hits_all:
                        self._ack_gone_entries(owned_gone_entries)
                        continue
                    files = list(
                        dict.fromkeys(
                            str(hit.orig_file_path) for hit in hits_regular
                        )
                    )
                    black = list(
                        dict.fromkeys(
                            str(hit.orig_file_path) for hit in hits_black
                        )
                    )
            await self._start_hook(cleanup_id, started, user_hits_all)
            # files whose running process is stopped are cleaned first, in
            # their own small pass, and never a second time
            stop_cancelled = False
            try:
                (
                    result,
                    cleaned_first,
                    error,
                    cmd,
                ) = await self._stop_running_processes(
                    user,
                    hits_regular,
                    hits_black,
                    standard_only,
                    cause,
                    initiator,
                    gone,
                    gone_entries=owned_gone_entries,
                    run_as=identity,
                    **(
                        {"copy_counts": copy_counts, "curator": curator}
                        if curator
                        else {}
                    ),
                )
            except _StopCleanupCancelled as cancelled:
                state = cancelled.state
                result, cleaned_first, error, cmd = (
                    state["result"],
                    state["cleaned"],
                    state["error"],
                    state["cmd"],
                )
                stop_cancelled = True
            files = [f for f in files if f not in cleaned_first]
            black = [f for f in black if f not in cleaned_first]
            if (files or black) and not stop_cancelled:
                rest, rest_error, cmd = await self._cleaner.start(
                    user,
                    files,
                    soft=Config.CLEANUP_TRIM,
                    blacklist=black,
                    standard_only=standard_only,
                    **({"run_as": identity} if identity else {}),
                    **(
                        {"blacklist_hashes": hashes}
                        if (
                            hashes := blacklist_hashes.from_hits(user_hits_all)
                        )
                        else {}
                    ),
                    **(
                        {"sa_verdicts": verdicts}
                        if (verdicts := sa_verdicts.from_hits(user_hits_all))
                        else {}
                    ),
                )
                if cleaned_first:
                    rest.update(result)
                    rest.attempts.update(getattr(result, "attempts", {}))
                    error = rest_error or error
                else:
                    error = rest_error
                result = rest
            reported_hits = user_hits_all
            if stop_cancelled:
                reported_hits = [
                    hit
                    for hit in user_hits_all
                    if str(hit.orig_file_path) in cleaned_first
                ]
                unattempted = [
                    hit for hit in user_hits_all if hit not in reported_hits
                ]
                unattempted.extend(unvisited_hits(user_index))
                requeue(unattempted)
                if not reported_hits:
                    raise asyncio.CancelledError
            message = MessageType.MalwareCleanup(
                hits=reported_hits,
                result=result,
                cleanup_id=cleanup_id,
                started=started,
                error=error,
                cause=cause,
                initiator=initiator,
                post_action=post_action,
                scan_id=scan_id,
                args=cmd,
                cleanup_user=user,
            )
            if stop_cancelled:
                task = asyncio.ensure_future(
                    self._sink.process_message(message)
                )
                while not task.done():
                    try:
                        await asyncio.shield(task)
                    except asyncio.CancelledError:
                        pass
                task.result()
                raise asyncio.CancelledError
            await self._sink.process_message(message)

    def _put_back_batch(self, batch, settled, step, user, count, exc, *scope):
        """An account's database step stayed locked: its files not
        *settled* go back to the proxy once this run is over (now, the run's
        proxy flush would take them again)."""
        settled_ids = {id(hit) for hit in settled}
        pending = [hit for hit in batch if id(hit) not in settled_ids]
        self._put_back.append((pending, *scope))
        _not_recorded(step, user, count, len(pending), exc)

    @staticmethod
    async def _drop_ignored(hits, settled):
        """*hits* less the ignored ones, which go back to found (recorded
        in *settled*). Only a lock that outlives the retries is raised."""
        kept = []
        for hit in hits:
            try:
                if await MalwareIgnorePath.is_path_ignored(hit.orig_file):
                    # Transition to a terminal state instead of silently skipping
                    await store_db.write(
                        MalwareHit.set_status,
                        [hit],
                        MalwareHitStatus.FOUND,
                        same_detection=True,
                    )
                    settled.append(hit)
                else:
                    kept.append(hit)
            except store_db.StoreBusyError:
                raise
            except Exception as exc:
                # be conservative: if ignore check fails, do not drop the hit
                logger.exception(
                    "Ignore check failed for file %s: %s; keeping hit",
                    hit.orig_file,
                    exc,
                )
                kept.append(hit)
        return kept

    async def _stop_running_processes(
        self,
        user,
        hits_regular,
        hits_black,
        standard_only,
        cause,
        initiator,
        gone=(),
        gone_entries=(),
        run_as=None,
        copy_counts=None,
        curator=None,
    ):
        """Stop the processes that run or hold the files about to be
        cleaned. The KILL candidates' files are cleaned here, in their own
        small pass, so a process is only killed once its file is clean.
        *gone* are hits whose file no longer exists: their processes are
        still looked for by the recorded inode; the cleaner never sees
        those files (the stop step kills only when the held inode has no
        link left on disk).

        Returns (result of that pass, the files it cleaned, error, cmd)."""
        if curator is None:
            curator = await curator_version.selected_curator()
        capable = await curator_version.system_owned_cleanup_available(curator)
        empty = (CleanupResult(), set(), None, [])
        if self._stop_controller is None or not stop_mode.evaluation_wanted():
            self._ack_gone_entries(gone_entries)
            return empty
        # The cleaner may replace a numeric account with its NSS username
        # before this boundary.  Authorize each immutable hit owner instead
        # of the mutable dispatch name, including retry-only gone entries.
        if copy_counts is None:
            try:
                copy_counts = await _root_cleanup_copy_counts(
                    [
                        *hits_regular,
                        *hits_black,
                        *gone,
                        *(entry[0] for entry in gone_entries),
                    ],
                    capable,
                    curator=curator,
                )
            except store_db.StoreBusyError as exc:
                # Gone-only retries retain their owned evidence for the next
                # run. Normal batches supply the already checked counts.
                _not_recorded(
                    "gone root-copy admission", user, len(gone), 0, exc
                )
                return empty
        refused_entries = [
            entry
            for entry in gone_entries
            if _refused_by_owner_guard(
                entry[0], capable, copy_counts=copy_counts, curator=curator
            )
        ]
        if refused_entries:
            self._ack_gone_entries(refused_entries)
        hits_regular = [
            hit
            for hit in hits_regular
            if not _refused_by_owner_guard(
                hit, capable, copy_counts=copy_counts, curator=curator
            )
        ]
        hits_black = [
            hit
            for hit in hits_black
            if not _refused_by_owner_guard(
                hit, capable, copy_counts=copy_counts, curator=curator
            )
        ]
        gone = [
            hit
            for hit in gone
            if not _refused_by_owner_guard(
                hit, capable, copy_counts=copy_counts, curator=curator
            )
        ]
        file_hits = [
            h
            for h in hits_regular + hits_black + list(gone)
            if h.resource_type == MalwareScanResourceType.FILE.value
        ]
        if not file_hits:
            self._ack_gone_entries(gone_entries)
            return empty
        self._note_gone_attempts(gone_entries)
        state = {
            "result": CleanupResult(),
            "cleaned": set(),
            "error": None,
            "cmd": [],
        }
        if is_uid(user):
            # The normal cleaner path applies this gate before replacing a
            # numeric owner with its NSS name.  Gone-only batches reach this
            # common boundary directly and must use the same eligibility.
            if not LicenseCLN.is_unlimited():
                self._ack_gone_entries(gone_entries)
                return empty
            try:
                user = await get_username_by_uid(user) or user
            except asyncio.CancelledError:
                raise _StopCleanupCancelled(state) from None
        cleaner_paths = {
            hit.orig_file: str(hit.orig_file_path)
            for hit in hits_regular + hits_black
        }
        black_files = {cleaner_paths[h.orig_file] for h in hits_black}
        gone_ids = frozenset(h.id for h in gone)

        async def clean(paths):
            by_path = {path: cleaner_paths.get(path, path) for path in paths}
            regular = sorted(
                {f for f in by_path.values() if f not in black_files}
            )
            black = sorted({f for f in by_path.values() if f in black_files})
            # claimed before the pass runs: a pass that fails or is
            # cancelled must not be repeated by the normal pass while the
            # curator may still be at work on these files
            state["cleaned"].update(by_path.values())
            cleaner = asyncio.create_task(
                self._cleaner.start(
                    user,
                    regular,
                    soft=Config.CLEANUP_TRIM,
                    blacklist=black or None,
                    standard_only=standard_only,
                    **({"run_as": run_as} if run_as else {}),
                    **(
                        {"blacklist_hashes": hashes}
                        if (hashes := blacklist_hashes.from_hits(hits_black))
                        else {}
                    ),
                    **(
                        {"sa_verdicts": verdicts}
                        if (verdicts := sa_verdicts.from_hits(hits_regular))
                        else {}
                    ),
                )
            )
            cancelled = False
            grace_end = None
            loop = asyncio.get_event_loop()
            while not cleaner.done():
                try:
                    if grace_end is None:
                        await asyncio.shield(cleaner)
                    elif loop.time() < grace_end:
                        await asyncio.wait_for(
                            asyncio.shield(cleaner), grace_end - loop.time()
                        )
                    else:
                        # the curator is terminated, so no pass of it can
                        # overlap a later cleanup of these files
                        cleaner.cancel()
                        await asyncio.wait({cleaner})
                except asyncio.CancelledError:
                    cancelled = True
                    if grace_end is None:
                        grace_end = loop.time() + SMALL_PASS_GRACE
                except asyncio.TimeoutError:
                    continue
                except Exception:
                    break
            if cleaner.cancelled():
                # nothing was cleaned: the files are cleaned after restart
                state["cleaned"].difference_update(by_path.values())
                raise asyncio.CancelledError
            try:
                result, error, cmd = cleaner.result()
            except Exception:
                if cancelled:
                    logger.exception(
                        "The owned cleaner failed while stop cleanup was"
                        " being cancelled"
                    )
                    raise asyncio.CancelledError from None
                raise
            reported = dict(result)
            attempts = dict(getattr(result, "attempts", {}))
            # CleanupResult lookup uses the DB path, not the cleaner operand.
            for original, file in cleaner_paths.items():
                if file in result:
                    reported[original] = result[file]
                if file in attempts:
                    attempts[original] = attempts[file]
            for file in cleaner_paths.values():
                if file not in cleaner_paths:
                    reported.pop(file, None)
                    attempts.pop(file, None)
            state["result"].update(reported)
            state["result"].attempts.update(attempts)
            state["error"] = error
            state["cmd"] = cmd
            outcomes = {
                path: cleaner_outcome(result[file] if file in result else None)
                for path, file in by_path.items()
            }
            if cancelled:
                raise asyncio.CancelledError
            return outcomes

        try:
            # only a run that may kill needs them; without them (None)
            # nothing is stopped in the run, also when a rule turns KILL
            # during the search
            veto_hits = None
            if stop_mode.base_mode()[0] is stop_mode.Mode.KILL:
                veto_hits = await self._loop.run_in_executor(
                    None,
                    _other_detected_hits,
                    {h.owner for h in file_hits} | {h.user for h in file_hits},
                    {h.id for h in file_hits},
                )
            out = await self._stop_controller.evaluate(
                file_hits,
                clean,
                gone_ids=gone_ids,
                user=user,
                veto_hits=veto_hits,
            )
            # reported once per cleanup run, with the other accounts' steps
            self._run_reports.append((out, file_hits, cause, initiator))
            if out.cancelled:
                # One hit may be linked to more than one process. A result
                # for one pid does not prove that cancellation did not skip
                # another, and deleted database rows cannot recreate this
                # work. Retain the exact owned batch for the next tick.
                raise _StopCleanupCancelled(state)
            if getattr(out, "skipped", False):
                # not an attempt: an earlier search is still stuck
                self._forget_gone_attempt(gone_entries)
            if getattr(out, "timed_out", False):
                # nothing was looked at: a gone file's recorded inode is the
                # only way to find its holder again, so it is kept for the
                # next tick (bounded by its attempts)
                return (
                    state["result"],
                    state["cleaned"],
                    state["error"],
                    state["cmd"],
                )
            # Evaluation owns these exact in-memory entries. Once it has
            # completed, later reporting failure must not repeat a
            # physical stop or its history on the next cleanup tick.
            by_pid = {}
            for evaluation in out.evaluations:
                by_pid.setdefault(evaluation.result.pid, []).append(evaluation)
            deferred_hit_ids = set()
            retryable = {
                RESPAWNING,
                STOP_CAP,
                COULD_NOT_FREEZE,
                FREEZE_TIMEOUT,
                READ_FAILED,
            }
            for evaluations in by_pid.values():
                intended = [
                    evaluation
                    for evaluation in evaluations
                    if evaluation.mode is stop_mode.Mode.KILL
                ]
                deferred = any(
                    evaluation.result.reason in retryable
                    for evaluation in intended
                ) or any(
                    not evaluation.result.file_gone
                    and evaluation.result.reason == FILE_NOT_CLEANED
                    for evaluation in evaluations
                )
                if deferred:
                    deferred_hit_ids.update(
                        evaluation.finding.hit_id
                        for evaluation in intended
                        if evaluation.result.file_gone
                    )
            self._ack_gone_entries(
                [
                    entry
                    for entry in gone_entries
                    if entry[0].id not in deferred_hit_ids
                ]
            )
        except asyncio.CancelledError:
            raise _StopCleanupCancelled(state) from None
        except Exception:
            logger.exception("Stopping processes during cleanup failed")
        return state["result"], state["cleaned"], state["error"], state["cmd"]

    def _note_gone_attempts(self, entries):
        owned = {id(entry) for entry in self._gone_pending}
        seen = set()
        for entry in entries:
            key = id(entry)
            if key not in owned or key in seen:
                continue
            seen.add(key)
            prior = self._gone_attempts.get(key)
            attempts = (
                prior[1] if prior is not None and prior[0] is entry else 0
            )
            self._gone_attempts[key] = (entry, attempts + 1)

    def _forget_gone_attempt(self, entries):
        for entry in entries:
            state = self._gone_attempts.get(id(entry))
            if state is not None and state[0] is entry:
                attempts = state[1] - 1
                if attempts > 0:
                    self._gone_attempts[id(entry)] = (entry, attempts)
                else:
                    del self._gone_attempts[id(entry)]

    def _expire_gone_entries(self):
        owned = {id(entry): entry for entry in self._gone_pending}
        self._gone_attempts = {
            key: state
            for key, state in self._gone_attempts.items()
            if key in owned and state[0] is owned[key]
        }
        expired = {
            key
            for key, (_, attempts) in self._gone_attempts.items()
            if attempts >= MAX_GONE_STOP_ATTEMPTS
        }
        if expired:
            self._gone_pending[:] = [
                entry
                for entry in self._gone_pending
                if id(entry) not in expired
            ]
            for key in expired:
                self._gone_attempts.pop(key, None)
            logger.info(
                "Stopped retrying %d missing-file process checks after %d"
                " attempts",
                len(expired),
                MAX_GONE_STOP_ATTEMPTS,
            )

    def take_gone_hits(self):
        """The hits whose file was already gone at the storage step, handed
        over once to the run's stop step."""
        pending, self._gone_pending = self._gone_pending, []
        self._gone_attempts.clear()
        return [hit for hit, _, _ in pending]

    def _take_gone_entries(self):
        self._expire_gone_entries()
        return list(self._gone_pending)

    def _ack_gone_entries(self, entries):
        """Forget only entries deliberately evaluated by this run."""
        for entry in entries:
            self._gone_attempts.pop(id(entry), None)
            for index, pending in enumerate(self._gone_pending):
                if pending is entry:
                    self._gone_pending.pop(index)
                    break

    async def _stop_for_gone_hits(self, entries):
        """The stop step for files already gone at the storage step, per
        owner and cleanup cause, without any cleaner pass."""
        groups = {}
        for hit, cause, initiator in entries:
            groups.setdefault((hit.owner, cause, initiator), []).append(hit)
        for (user, cause, initiator), hits in groups.items():
            owned = [
                entry
                for entry in entries
                if (entry[0].owner, entry[1], entry[2])
                == (user, cause, initiator)
            ]
            await self._stop_running_processes(
                user,
                [],
                [],
                None,
                cause,
                initiator,
                hits,
                gone_entries=owned,
            )

    async def _report_run(self):
        """One report per cleanup run: the stop steps of all its accounts
        in one MALWARE_PROCESS_STOP_LIST (process items plus one counters
        item with runs 1) or, when nothing was linked, one roll-up entry;
        the admin-visible record per step in LOG and KILL. The record is
        written first: a failed or cancelled send never loses it."""
        parts, self._run_reports = self._run_reports, []
        # a search that gave up or was skipped examined nothing, but the
        # run is counted
        parts = [
            p
            for p in parts
            if p[0].evaluations or p[0].examined or p[0].timed_out
        ]
        if not parts:
            return
        merged = stop_report.merge_run(
            [(out, hits) for out, hits, _, _ in parts]
        )
        loop = asyncio.get_event_loop()
        for out, hits, cause, initiator in parts:
            try:
                await loop.run_in_executor(
                    None,
                    _record_stop_history,
                    out,
                    hits,
                    cause,
                    initiator,
                )
            except asyncio.CancelledError:
                raise
            except Exception:
                logger.exception(
                    "Writing one account's process-stop history failed"
                )
        items = []
        kernel, panel, btime = (
            _kernel(),
            HostingPanel().NAME,
            stop_proc.boot_time(),
        )
        users = {h.owner for _, hits, _, _ in parts for h in hits}
        try:
            homes = await asyncio.get_event_loop().run_in_executor(
                None, _homes, users
            )
        except Exception as e:
            logger.debug("Resolving cleanup account homes failed: %r", e)
            homes = {user: "" for user in users}
        for out, hits, _, _ in parts:
            built = stop_report.build_items(
                out,
                hits,
                home_of=homes,
                kernel=kernel,
                panel=panel,
                btime=btime,
            )
            items.extend(i for i in built if i["record_type"] == "process")
        items.append(stop_report.counters_item(merged, kernel, panel))
        await stop_report.report_run(self._sink, items)
        active_now = stop_mode.base_mode().mode in (
            stop_mode.Mode.LOG,
            stop_mode.Mode.KILL,
        )
        if active_now or any(
            ev.mode in (stop_mode.Mode.LOG, stop_mode.Mode.KILL)
            for ev in merged.evaluations
        ):
            stop_report.log_timings(merged)

    async def _cleanup(self):
        if self._running:
            return
        # Expiry precedes the tracked work path so a failed backlog can idle.
        self._expire_gone_entries()
        if not self._proxy.hits and not self._gone_pending:
            await self._retry_left_overs_coalesced()
            if not self._proxy.hits and not self._gone_pending:
                self._proxy.reset()
            return
        self._running = True
        cleanup_cancelled = False
        cleanup_error = None

        with inactivity.track.task("cleanup"):
            try:
                if self._stop_controller is not None:
                    self._stop_controller.begin_run()  # the run's stop cap
                await self._retry_left_overs_coalesced()
                # files already gone at the storage step join the stop
                # step of the owner's batch of this run; the rest (owners
                # without a batch this run) get their own stop step
                gone_entries = self._take_gone_entries()
                data = self._cloud_assisted_hits()
                for (
                    all_hits,
                    blacklist,
                    cause,
                    initiator,
                    post_action,
                    scan_id,
                    standard_only,
                ) in data:
                    await self._clean_files(
                        all_hits,
                        blacklist=blacklist,
                        cause=cause,
                        initiator=initiator,
                        post_action=post_action,
                        scan_id=scan_id,
                        standard_only=standard_only,
                        gone_entries=gone_entries,
                    )
                if gone_entries:
                    await self._stop_for_gone_hits(gone_entries)
            except asyncio.CancelledError:
                cleanup_cancelled = True
                raise
            except Exception as error:
                cleanup_error = error
                raise
            finally:
                put_back, self._put_back = self._put_back, []
                for args in put_back:
                    self._add_to_proxy(*args)
                # review 6 L7: a later account's failure never loses the
                # history and telemetry of kills already made
                try:
                    report_task = asyncio.ensure_future(self._report_run())
                    report_cancelled = cleanup_cancelled
                    report_error = None
                    while not report_task.done():
                        try:
                            await asyncio.shield(report_task)
                        except asyncio.CancelledError:
                            report_cancelled = True
                        except Exception as error:
                            report_error = error
                            break
                    current = asyncio.current_task()
                    if current is not None and current.cancelling():
                        report_cancelled = True
                    if report_error is None:
                        try:
                            report_task.result()
                        except Exception as error:
                            report_error = error
                    if report_error is not None:
                        if not report_cancelled and cleanup_error is None:
                            raise report_error
                        logger.error(
                            "Could not report process-stop results %s",
                            "during cancellation"
                            if report_cancelled
                            else "after cleanup failed",
                            exc_info=(
                                type(report_error),
                                report_error,
                                report_error.__traceback__,
                            ),
                        )
                    if report_cancelled:
                        raise asyncio.CancelledError
                finally:
                    if self._stop_controller is not None:
                        self._stop_controller.end_run()  # review 6 L9
                    self._running = False

    async def _retry_left_overs(self):
        """Every cleanup tick: the cron re-enables a slow crontab left
        pending and the frozen processes whose SIGCONT failed at start are
        tried again, each only while its state file exists (it is removed
        once empty), so an idle tick costs a stat; once a minute, in the
        executor, so are the files of cleanup runs whose cleaner outlived
        the agent."""
        now = self._loop.time()
        if (
            now - self._cron_checked_at >= CRON_RETRY_INTERVAL
            and stop_cron.DISABLED_PATH.exists()
        ):
            # pending re-enables are retried at most once a minute: the
            # store can hold hundreds of disabled lines
            self._cron_checked_at = now
            try:
                await stop_cron.retry_pending_reenables()
            except Exception:
                logger.exception("Could not retry the pending cron re-enables")
        if self._runs_swept_at is None:
            self._runs_swept_at = now
        elif now - self._runs_swept_at >= RUNS_RETRY_INTERVAL:
            # a cleaner the previous agent left running may have exited
            self._runs_swept_at = now
            await self._loop.run_in_executor(None, sweep_interrupted_runs)
        if not stop_registry.REGISTRY_PATH.exists():
            return
        future = self._loop.run_in_executor(None, release_frozen_orphans)
        cancelled = False
        while not future.done():
            try:
                await asyncio.shield(future)
            except asyncio.CancelledError:
                cancelled = True
        released = future.result()
        if released:
            report_task = asyncio.ensure_future(
                stop_report.report_released(
                    self._sink,
                    released,
                    kernel=_kernel(),
                    panel=HostingPanel().NAME,
                )
            )
            while not report_task.done():
                try:
                    await asyncio.shield(report_task)
                except asyncio.CancelledError:
                    cancelled = True
            try:
                report_task.result()
            except Exception:
                logger.exception("Could not report the processes released")
        if cancelled:
            raise asyncio.CancelledError

    async def _retry_left_overs_coalesced(self):
        task = self._retry_task
        if task is None or task.done():
            task = asyncio.ensure_future(self._retry_left_overs())
            self._retry_task = task
        try:
            await asyncio.shield(task)
        finally:
            if task.done() and self._retry_task is task:
                self._retry_task = None

    @recurring_check(1)
    async def cleanup(self):
        try:
            await self._cleanup()
        except asyncio.CancelledError:
            # Counters are already durable.  Do not delay or replace caller
            # cancellation with a new transport operation.
            raise
        except Exception:
            # Earlier completed runs have already persisted their counters;
            # a later account failure must not indefinitely postpone the
            # hourly send.
            try:
                await stop_report.flush_rollup(self._sink)
            except asyncio.CancelledError:
                raise
            except Exception:
                logger.exception(
                    "Could not flush process-stop counters after cleanup"
                )
            raise
        else:
            # Preserve the pre-existing successful-tick behavior: a send
            # failure remains visible to the recurring-check wrapper.
            await stop_report.flush_rollup(self._sink)


class ResultProcessor(MessageSink, MessageSource):
    SCOPE = Scope.AV

    async def create_sink(self, loop):
        pass

    async def create_source(self, loop, sink):
        self._sink = sink

    @staticmethod
    async def _set_hit_status(
        hits: List[MalwareHit], status: str, cleaned_at=None
    ):
        # only the rows still holding these detections: one deleted while
        # the cleaner ran may have lent its id to a new row
        await store_db.write(
            MalwareHit.set_status,
            hits,
            status,
            cleaned_at,
            same_detection=True,
        )
        for hit in hits:
            hit.status = status
            hit.cleaned_at = cleaned_at

    @expect(MessageType.MalwareCleanup)
    async def store_result(self, message):
        hits: List[MalwareHit] = message["hits"]
        result: CleanupResult = message["result"]
        cause = message.get("cause")
        initiator = message.get("initiator")
        now = time.time()

        processed = [hit for hit in hits if hit in result]
        unprocessed = [hit for hit in hits if hit not in result]
        if processed:
            await result.mark_unchanged(
                {
                    hit.orig_file: hit.hash
                    for hit in processed
                    if not hit.orig_file_path.is_remote
                }
            )

        error = message.get("error")
        if unprocessed:
            sample = [h.orig_file for h in unprocessed[:3]]
            logger.warning(
                "Cleanup: %d/%d hits unmatched in result"
                " (error=%r, sample=%s)",
                len(unprocessed),
                len(hits),
                error,
                sample,
            )
        if error and result:
            logger.warning(
                "Cleanup: partial failure — error=%r"
                " but %d result entries present",
                error,
                len(result),
            )

        not_exist = []
        async for hit in nice_iterator(processed, chunk_size=100):
            # in case if procu2.php tries to clean user file in root dirs,
            # it will be marked as non-existent due to 'Permission denied'
            # error which confuses users, so consider it as unable to cleanup.
            if result[hit].not_exist():  # pragma: no cover
                if await safe_exists(hit.orig_file_path):
                    unprocessed.append(hit)
                else:
                    not_exist.append(hit)

        # DEF-40165: files missing from procu2 result that no longer exist
        # on disk must be removed instead of being restored to FOUND,
        # otherwise they get re-dispatched for cleanup indefinitely.
        still_unprocessed = []
        unprocessed_existence = await gather_exists(
            [h.orig_file_path for h in unprocessed]
        )
        for hit, exists in zip(unprocessed, unprocessed_existence):
            if not exists:
                not_exist.append(hit)
            else:
                still_unprocessed.append(hit)
        unprocessed = still_unprocessed

        # a refusal names its reason: no cleaner ran to report one
        refusal = message.get("refusal_reason")
        details = {
            hit.orig_file: local_details(
                {
                    **getattr(result, "attempts", {}).get(
                        hit.orig_file,
                        getattr(result, "attempts", {}).get(
                            str(hit.orig_file_path), {}
                        ),
                    ),
                    **result.get(hit.orig_file, {}),
                    **({"reason": refusal} if refusal else {}),
                },
                message.get("cleanup_user", hit.user),
                bool(error),
            )
            for hit in hits
        }
        message["cleanup_details"] = {}

        def record_diagnostics(outcome_hits, event):
            for hit in outcome_hits:
                detail = details[hit.orig_file]
                message["cleanup_details"][hit.orig_file] = detail
                log_diagnostic(logger, hit.orig_file, detail, event)

        # the hits whose outcome is not recorded yet, by identity
        pending = {id(hit): hit for hit in hits}

        def recorded(outcome_hits):
            for hit in outcome_hits:
                pending.pop(id(hit), None)

        # the failed files go to MRS whatever is recorded
        failed = [hit for hit in processed if result[hit].is_failed()]
        # the plugins after this one see these outcomes whatever is recorded
        outcomes = (
            (
                [
                    hit
                    for hit in processed
                    if result[hit].requires_myimunify_protection()
                ],
                MalwareHitStatus.CLEANUP_REQUIRES_MYIMUNIFY_PROTECTION,
            ),
            (
                [hit for hit in processed if result[hit].is_cleaned()],
                MalwareHitStatus.CLEANUP_DONE,
            ),
            (
                [hit for hit in processed if result[hit].is_removed()],
                MalwareHitStatus.CLEANUP_REMOVED,
            ),
        )
        requires_myimunify_protection, cleaned, removed = (
            outcome_hits for outcome_hits, _ in outcomes
        )
        step = "unable history"
        try:
            record_diagnostics(unprocessed, "unable_to_cleanup")
            await _with_history(
                MalwareAction.cleanup_unable,
                unprocessed,
                cause=cause,
                initiator=initiator,
                details=details,
            )

            step = "requires-MyImunify history"
            await _with_history(
                MalwareAction.cleanup_requires_myimunify_protection,
                requires_myimunify_protection,
                cause=cause,
                initiator=initiator,
            )
            step = "requires-MyImunify status"
            await self._set_hit_status(
                requires_myimunify_protection,
                MalwareHitStatus.CLEANUP_REQUIRES_MYIMUNIFY_PROTECTION,
                now,
            )
            recorded(requires_myimunify_protection)

            record_diagnostics(failed, "failed_to_cleanup")
            step = "failed history"
            await _with_history(
                MalwareAction.cleanup_failed,
                failed,
                cause=cause,
                initiator=initiator,
                details=details,
            )

            step = "cleaned history"
            await _with_history(
                MalwareAction.cleanup_done,
                cleaned,
                cause=cause,
                initiator=initiator,
            )
            step = "cleaned status"
            await self._set_hit_status(
                cleaned, MalwareHitStatus.CLEANUP_DONE, now
            )
            recorded(cleaned)

            step = "removed history"
            await _with_history(
                MalwareAction.cleanup_removed,
                removed,
                cause=cause,
                initiator=initiator,
            )
            step = "removed status"
            await self._set_hit_status(
                removed, MalwareHitStatus.CLEANUP_REMOVED, now
            )
            recorded(removed)

            # a result row matching no outcome predicate (e.g. an error code
            # this agent does not know yet) must still reach a terminal
            # state, otherwise the hit stays in cleanup_started until agent
            # restart
            unclassified = [
                hit
                for hit in processed
                if not (
                    result[hit].not_exist()
                    or result[hit].requires_myimunify_protection()
                    or result[hit].is_failed()
                    or result[hit].is_cleaned()
                    or result[hit].is_removed()
                )
            ]
            if unclassified:
                logger.warning(
                    "Cleanup: %d/%d hits with unclassified result entries"
                    " (sample=%s)",
                    len(unclassified),
                    len(hits),
                    [(h.orig_file, result[h]["e"]) for h in unclassified[:3]],
                )
                record_diagnostics(unclassified, "unable_to_cleanup")
                step = "unclassified history"
                await _with_history(
                    MalwareAction.cleanup_unable,
                    unclassified,
                    cause=cause,
                    initiator=initiator,
                    details=details,
                )

            step = "gone-file delete"
            await store_db.write(
                MalwareHit.delete_instances, not_exist, same_detection=True
            )
            step = "gone-file history"
            await _with_history(
                MalwareAction.not_exist,
                not_exist,
                cause=cause,
                initiator=initiator,
            )
            recorded(not_exist)

            step = "status reset"
            for status, hit_list in _group_by_status(
                unprocessed, failed, unclassified
            ).items():
                await self._set_hit_status(hit_list, status)
                recorded(hit_list)
        except store_db.StoreBusyError as exc:
            # The files are cleaned or not on disk whatever is recorded; the
            # hits not recorded stay cleanup_started until the agent starts
            # again. The plugins after this one still get the message.
            _not_recorded(
                "result " + step,
                message.get("cleanup_user") or _owners(hits),
                len(pending),
                0,
                exc,
            )
            for outcome_hits, status in outcomes:
                for hit in outcome_hits:
                    if id(hit) in pending:
                        hit.status = status
                        hit.cleaned_at = now

        await self.send_failed_to_cleanup_hits_to_mrs(failed)

        return message

    async def send_failed_to_cleanup_hits_to_mrs(self, failed_to_cleanup_hits):
        if failed_to_cleanup_hits:
            await self._sink.process_message(
                MessageType.MalwareMRSUpload(
                    hits=[
                        malware_response.HitInfo(hit.orig_file, hit.hash)
                        for hit in failed_to_cleanup_hits
                    ],
                    upload_reason="cleanup_failure_current",
                )
            )
            await self._sink.process_message(
                MessageType.MalwareMRSUpload(
                    hits=[
                        malware_response.HitInfo(
                            str(CleanupStorage.get_hit_store_path(hit)),
                            hit.hash,
                        )
                        for hit in failed_to_cleanup_hits
                    ],
                    upload_reason="cleanup_failure_original",
                )
            )


class StorageController(MessageSink):
    """Remove old backed up files from storage"""

    def __init__(self):
        self._clear_task = None
        self._clear_lock = asyncio.Lock()
        self._keep = Config.CLEANUP_KEEP

    async def create_sink(self, loop):
        self._clear_task = loop.create_task(self.daily_clear())

    async def shutdown(self):
        if self._clear_task:
            await safe_cancel_task(self._clear_task)

    async def _clear(self):
        async with self._clear_lock:
            now = time.time()
            keep_hits = now - self._keep * DAY
            keep_orig = now - (self._keep + 1) * DAY  # keep files one more day
            step = "hook registration lookup"

            async def hook_db_write(operation):
                nonlocal step
                step = f"hook database operation {operation.__name__}"
                return await store_db.write(operation)

            try:
                if await store_db.write(hooked):
                    await delete_and_report(
                        EXPIRED,
                        MalwareHit.cleaned_at < keep_hits,
                        db_write=hook_db_write,
                    )
                step = "final expired-hit delete"
                await store_db.write(
                    lambda: MalwareHit.delete()
                    .where(MalwareHit.cleaned_at < keep_hits)
                    .execute()
                )
            except store_db.StoreBusyError as exc:
                logger.error(
                    "Cleanup storage database busy during %s; retry next"
                    " run: %s",
                    step,
                    exc,
                )
                return
            cleared = await CleanupStorage.clear(keep_orig)
            if cleared:
                logger.info(
                    "Cleanup storage have cleaned. Files removed: %s", cleared
                )

    @expect(MessageType.ConfigUpdate)
    @utils.log_error_and_ignore()
    async def config_updated(self, _):
        if self._keep != Config.CLEANUP_KEEP:
            self._keep = Config.CLEANUP_KEEP
            await self._clear()

    @recurring_check(
        check_lock,
        check_period_first=True,
        check_lock_period=DAY,
        lock_file=LOCK_FILE,
    )
    async def daily_clear(self):
        await self._clear()


def decide_if_standard_signatures_only(user, standard_only):
    """Root user or user with MyImunify can use advanced signatures"""

    if not MyImunifyConfig.ENABLED:
        return False

    if user is None or user == "root" or myimunify_protection_enabled(user):
        return standard_only

    return True


class ResultProcessorIm360(ResultProcessor):
    """Imunify360 specialization of ResultProcessor, which removes all
    cleaned and removed files from HackerTrap
    """

    SCOPE = Scope.IM360

    @expect(MessageType.MalwareCleanup)
    async def store_result(self, message):
        message = await super().store_result(message)
        to_remove = [
            Path(hit)
            for hit, state in message["result"].items()
            if (state.is_cleaned() or state.is_removed())
        ]
        await HackerTrapHitsSaver.update_sa_hits([], to_remove)


#: Batch children are named so: a message about one is told from a legacy
#: restore's by its id alone, whatever the agent remembers or no longer
#: has a row for (DEF-49301).
BATCH_CHILD_PREFIX = "mdsbatch-"
_BATCH_CHILD = re.compile(re.escape(BATCH_CHILD_PREFIX) + r"[0-9a-f]{32}")


def _new_batch_child() -> str:
    return BATCH_CHILD_PREFIX + uuid.uuid4().hex


def _is_batch_child(child) -> bool:
    return isinstance(child, str) and _BATCH_CHILD.fullmatch(child) is not None


class _TakenCompletions:
    """Ids of MDS operations whose completion has already been taken.

    Both the completion notification and the reconciler deliver a
    completion. The first delivery removes the run directory whatever the
    outcome, so a later one can only repeat the outcome or, finding no
    report, fail the operation queued next.
    """

    MAX_IDS = 256

    def __init__(self):
        self._ids: Dict[str, None] = {}

    def take(self, operation_id: str) -> bool:
        if operation_id in self._ids:
            return False
        self._ids[operation_id] = None
        if len(self._ids) > self.MAX_IDS:
            del self._ids[next(iter(self._ids))]
        return True


class CleanupDb(MessageSink):
    SCOPE = Scope.IM360

    def __init__(self):
        self._loop = None
        self._completions = _TakenCompletions()
        #: One queue run at a time: its start awaits the database.
        self._next_lock = asyncio.Lock()

    @staticmethod
    async def _start_cleaner(path, app_name):
        cleanup_id = uuid.uuid4().hex
        await MalwareDatabaseCleaner(cleanup_id, path, app_name).start()

    async def _cleanup_next(self):
        async with self._next_lock:
            # Claimed hits hold the queue and nothing resets database hits
            # under cleanup: once the claim is written its cleaner starts,
            # also when the caller is cancelled meanwhile.
            task = asyncio.ensure_future(self._take_next())
            try:
                await asyncio.shield(task)
            except asyncio.CancelledError:
                with suppress(Exception):
                    await asyncio.wait({task})
                raise

    async def _take_next(self):
        def _take_next():
            # The queue is serial by this status, so the next database and
            # its queued hits are read and claimed in one transaction
            # (atomic() begins IMMEDIATE): none if another cleanup holds the
            # queue or nothing is queued.
            with MalwareHit._meta.database.atomic():
                if (
                    MalwareHit.db_hits_under_cleanup().exists()
                    or (
                        next_hit := MalwareHit.db_hits_pending_cleanup()
                        .order_by(MalwareHit.timestamp.asc())
                        .first()
                    )
                    is None
                ):
                    return None
                # One --clean pass cleans the whole DB, so batch all its
                # pending hits.
                same_db_hits = list(
                    MalwareHit.db_hits_pending_cleanup().where(
                        (MalwareHit.orig_file == next_hit.orig_file)
                        & (MalwareHit.app_name == next_hit.app_name)
                    )
                )
                MalwareHit.set_status(
                    same_db_hits, MalwareHitStatus.CLEANUP_STARTED
                )
                return next_hit

        try:
            next_hit = await store_db.write(_take_next)
        except store_db.StoreBusyError as exc:
            # still queued: the next queue run takes them
            try:
                queued = await store_db.write(
                    lambda: list(MalwareHit.db_hits_pending_cleanup())
                )
            except store_db.StoreBusyError:
                queued = []
            _not_recorded(
                "database cleanup start",
                _databases(queued),
                len(queued),
                0,
                exc,
            )
            return
        if next_hit is None:
            return
        logger.info(
            "Cleaning hit: (%s::%s::%s)",
            next_hit.orig_file,
            next_hit.app_name,
            next_hit.type,
        )
        await self._start_cleaner(next_hit.orig_file, next_hit.app_name)

    async def create_sink(self, loop):
        self._loop = loop
        await self._cleanup_next()

    @expect(MessageType.MalwareCleanupTask)
    async def process_cleanup_task(self, message):
        def _queue():
            # read and queued in one transaction: a task running beside this
            # one sees them queued, not found
            with MalwareHit._meta.database.atomic():
                # by id, then only the rows still holding the requested
                # findings: a new MDS scan may have given a deleted hit's
                # id to another database's finding
                asked = {
                    hit.id: _database_finding(hit) for hit in message["hits"]
                }
                hits = [
                    hit
                    for hit in MalwareHit.refresh_hits(message["hits"])
                    if asked.get(hit.id) == _database_finding(hit)
                ]
                hits_to_clean = filter_cleanable(hits)
                db_hits = [
                    hit
                    for hit in hits_to_clean
                    if hit.resource_type == MalwareScanResourceType.DB.value
                ]
                if db_hits:
                    MalwareHit.set_status(
                        db_hits, MalwareHitStatus.CLEANUP_PENDING
                    )
                return db_hits

        try:
            db_hits = await store_db.write(_queue)
        except store_db.StoreBusyError as exc:
            # nothing is queued: the hits stay found
            asked = [
                hit
                for hit in message["hits"]
                if hit.resource_type == MalwareScanResourceType.DB.value
            ]
            _not_recorded(
                "database cleanup queue",
                _databases(asked),
                len(asked),
                0,
                exc,
            )
            return
        if not db_hits:
            return
        await self._cleanup_next()

    @expect(MessageType.MalwareCleanComplete)
    async def parse_cleanup_results(self, message):
        clean_id = message["scan_id"]
        if not self._completions.take(clean_id):
            logger.info(
                "Ignoring a duplicate completion of cleanup %s", clean_id
            )
            return
        detached_cleanup = MDSDetachedCleanup(clean_id)
        try:
            cleanup_outcome = await detached_cleanup.complete()
        except ScanAlreadyCompleteError:
            # This happens when AV is woken up by AiBolit. See DEF-11078.
            logger.warning(
                "Cannot complete cleanup %s, assuming it is already complete",
                clean_id,
            )
            return
        finally:
            shutil.rmtree(
                str(detached_cleanup.detached_dir), ignore_errors=True
            )
        await g.sink.process_message(cleanup_outcome)

    async def _finish(self, settle, step):
        """Settle the hits under cleanup in one step, write the history of
        those it fails, then run the queue. A step that stays locked leaves
        them under cleanup, which holds the queue (no later run takes
        them)."""
        try:
            leftover_hits = await store_db.write(settle)
        except store_db.StoreBusyError as exc:
            try:
                under_cleanup = await store_db.write(
                    lambda: list(MalwareHit.db_hits_under_cleanup())
                )
            except store_db.StoreBusyError:
                under_cleanup = []
            _not_recorded(
                step,
                _databases(under_cleanup),
                len(under_cleanup),
                0,
                exc,
            )
            return
        try:
            await _with_history(
                MalwareAction.cleanup_failed,
                leftover_hits,
                cause=None,
                initiator=None,
            )
        except store_db.StoreBusyError as exc:
            _not_recorded(
                step + " history",
                _databases(leftover_hits),
                len(leftover_hits),
                0,
                exc,
            )
        await self._cleanup_next()

    @expect(MessageType.MalwareDatabaseCleanup)
    async def update_cleaned_hits_status(
        self, message: MessageType.MalwareDatabaseCleanup
    ):
        now = time.time()

        def _settle():
            # read when the settle runs: a new scan may replace these rows
            # (and SQLite reuse their ids) while it waits
            with MalwareHit._meta.database.atomic():
                cleaned_hits = MalwareHit.db_hits_under_cleanup_in(
                    message.succeeded
                )
                failed_hits = MalwareHit.db_hits_under_cleanup_in(
                    message.failed
                )
                MalwareHit.set_status(
                    cleaned_hits, MalwareHitStatus.CLEANUP_DONE, now
                )
                MalwareHit.set_status(failed_hits, MalwareHitStatus.FOUND)
                # The queue is serial, so any hit still under cleanup
                # belongs to this operation; without a terminal status it
                # would block the queue forever. The sweep is deliberately
                # not scoped to this operation's (path, app): a run that
                # fails before reaching any database reports no hits at
                # all, and only an unscoped sweep finalizes its hits then.
                # The trade-off is that hits stranded in cleanup_started by
                # an unrelated earlier crash are finalized (and get a
                # failed_to_cleanup event) on this run's behalf.
                leftover_hits = list(MalwareHit.db_hits_under_cleanup())
                MalwareHit.set_status(leftover_hits, MalwareHitStatus.FOUND)
            return leftover_hits

        await self._finish(_settle, "database cleanup result")

    @expect(MessageType.MalwareDatabaseCleanupFailed)
    async def update_failed_hits_status(self, message):
        """
        Clear the queue when the cleanup fails,
        set hits' status back to infected
        """

        def _settle():
            # We assume here that all CLEANUP_STARTED hits are part of the
            # same cleanup operation; read when the settle runs
            with MalwareHit._meta.database.atomic():
                hits = list(MalwareHit.db_hits_under_cleanup())
                MalwareHit.set_status(hits, MalwareHitStatus.FOUND)
            return hits

        await self._finish(_settle, "database cleanup failure")

    @expect(MessageType.MalwareDatabaseCleanup)
    async def save_cleanup_events_in_history(
        self, message: MessageType.MalwareDatabaseCleanup
    ):
        cause = None
        initiator = None
        cleaned_hits = MalwareHit.get_db_hits(message.succeeded)
        failed_hits = MalwareHit.get_db_hits(message.failed)
        step_hits = [*cleaned_hits, *failed_hits]
        try:
            await _with_history(
                MalwareAction.cleanup_done,
                cleaned_hits,
                cause=cause,
                initiator=initiator,
            )
            step_hits = failed_hits
            await _with_history(
                MalwareAction.cleanup_failed,
                failed_hits,
                cause=cause,
                initiator=initiator,
            )
        except store_db.StoreBusyError as exc:
            # the statuses are still written, by update_cleaned_hits_status
            _not_recorded(
                "database cleanup history",
                _databases(step_hits),
                len(step_hits),
                0,
                exc,
            )


def _restore_event(hit: MalwareHit, event: str) -> dict:
    """The history row a database restore writes for *hit*, as
    ``MalwareAction.cleanup_restored_original`` and ``cleanup_failed_restore``
    write it, to be saved in the restore's own transaction."""
    return dict(
        event=event,
        path=hit.orig_file,
        app_name=hit.app_name,
        resource_type=MalwareScanResourceType.DB.value,
        file_owner=hit.owner,
        file_user=hit.user,
        initiator=UserType.ROOT,
        cause=MalwareScanType.MANUAL,
        ctime=int(time.time()),
        db_host=hit.db_host,
        db_port=hit.db_port,
        db_name=hit.db_name,
        table_name=None,
        table_field=None,
        table_row_inf=None,
        scan_id=None,
        signature_id=hit.signature_id,
    )


def _report(update, *args, **kwargs):
    """Update the database restore operations, which are a report of the
    restore: if their tables are locked or missing, the hits are still
    settled, their history written and the queue goes on. (Inside a batch
    transaction an update runs in a savepoint of its own, so an ordinary SQL
    failure rolls back that update only.)"""
    try:
        return update(*args, **kwargs)
    except Exception:
        logger.exception("Cannot update the database restore operations")


def _report_outcomes(restored, failed, applications=()):
    """Record the hits a restore settled, and end the operations nothing will
    settle further: those of the applications of the hits and of the
    *applications* the restore was for, which may have no hit left."""
    _report(operations.settle, restored, failed)
    for path, app_name in {
        *applications,
        *((hit.orig_file, hit.app_name) for hit in [*restored, *failed]),
    }:
        _report(operations.reap, path, app_name)


class RestoreOriginalDb(MessageSink):
    """The serial database restore queue.

    A signature restore is one legacy child. An application restore runs in
    batches of at most 100 cells, one guarded native child each, until the
    native report says nothing is left (``docs/MDS_RESTORE_CONTINUATION.md``).
    Its hits stay under restoration across the batches; their status and
    history are written once, with the last batch.
    """

    SCOPE = Scope.IM360
    #: A batch child's launches: the first and one relaunch for a lost result.
    MAX_BATCH_LAUNCHES = 2
    #: Seconds before a lost child that is still alive is looked at again.
    RECOVERY_RECHECK = 60
    #: Seconds ``shutdown`` waits for a recovery that is launching.
    RECHECK_FINISH_TIMEOUT = 10

    def __init__(self):
        self.loop = None
        self._completions = _TakenCompletions()
        #: Batch children whose completion is being accepted right now.
        self._accepting: set = set()
        #: Batch children being launched again right now.
        self._relaunching: set = set()
        #: One restore starts at a time: its start awaits the database.
        self._starting = asyncio.Lock()
        #: The pending looks at lost children that are still alive.
        self._rechecks: set = set()
        #: The timers that will start those looks.
        self._timers: set = set()
        #: The recoveries that only wait for the process table: a cancel
        #: costs them nothing, unlike one between a launch count and a spawn.
        self._waiting: set = set()
        #: Set by ``shutdown``: nothing is recovered or looked at again until
        #: the plugin is activated again.
        self._stopped = False

    async def _restore_next(self):
        # A launch that fails finishes its restore without starting the next
        # one, which this loop starts in the same frame: a long queue with
        # the route missing must not nest one launch inside another.
        while not self._stopped and not await self._restore_one():
            pass

    @staticmethod
    def _next_to_restore():
        """The oldest pending hit, unless a restore is under way."""
        if (
            MalwareHit.db_hits_under_restoration().exists()
            or MDSRestoreBatch.select()
            .where(MDSRestoreBatch.done == False)  # noqa: E712
            .exists()
        ):
            return None
        return (
            MalwareHit.db_hits_pending_cleanup_restore()
            .order_by(MalwareHit.timestamp.asc())
            .first()
        )

    async def _restore_one(self) -> bool:
        """Start the oldest pending restore: ``False`` when it failed to
        launch and the next one should be tried. One at a time: the probe
        and the start of a batch restore await, and the queue is serial only
        by the status and batch row the start writes."""
        async with self._starting:
            return await self._start_oldest()

    async def _start_oldest(self) -> bool:
        hit_to_restore = self._next_to_restore()
        if hit_to_restore is None:
            return True
        route = False
        if (
            hit_to_restore.status
            != MalwareHitStatus.CLEANUP_REMOTE_RESTORE_PENDING
        ):
            # An application restore chooses its route once, here (the
            # flags, then the probes); a signature restore is legacy and
            # probes nothing. The queue is serial only by the status and
            # batch row a restore writes. The probe awaits, so the queue is
            # chosen again after it, with no await until those are written:
            # a queue run meanwhile starts nothing beside this one.
            route = await guarded_route_available()
            if (hit_to_restore := self._next_to_restore()) is None:
                return True

        signature_id = (
            hit_to_restore.signature_id
            if hit_to_restore.status
            == MalwareHitStatus.CLEANUP_REMOTE_RESTORE_PENDING
            else None
        )

        logger.info(
            "Restoring from cleanup hit: (%s::%s::%s)",
            hit_to_restore.orig_file,
            hit_to_restore.app_name,
            signature_id,
        )
        if signature_id is None:
            if route:
                return await self._start_batches(
                    hit_to_restore.orig_file, hit_to_restore.app_name
                )
            # Without the guarded entry or the native journal (the released
            # ai-bolit and rustbolit), the restore takes the legacy route as
            # before batches: this hit alone, as that route restores a
            # limited number of cells per launch.
            logger.info(
                "The guarded MDS restore route is not installed, restoring"
                " %s (%s) through the legacy route",
                hit_to_restore.orig_file,
                hit_to_restore.app_name,
            )

        # Under restoration before the launch awaits, so that the queue,
        # run again meanwhile, does not start another restore beside it.
        MalwareHit.set_status(
            [hit_to_restore], MalwareHitStatus.CLEANUP_RESTORE_STARTED
        )
        if signature_id is None:
            _report(
                operations.mark_running,
                hit_to_restore.orig_file,
                hit_to_restore.app_name,
            )
        try:
            await MalwareDatabaseRestore(
                path=hit_to_restore.orig_file,
                app_name=hit_to_restore.app_name,
                signature_id=signature_id,
            ).restore()
        except BaseException:
            MalwareHit.set_status([hit_to_restore], hit_to_restore.status)
            raise
        return True

    async def _start_batches(self, path, app_name) -> bool:
        """Restore an application: its pending hits are this restore's."""

        def _start():
            with MDSRestoreBatch._meta.database.atomic():
                row = MDSRestoreBatch.create(
                    child_id=_new_batch_child(), path=path, app_name=app_name
                )
                MalwareHit.set_status(
                    MalwareHit.db_hits().where(
                        MalwareHit.orig_file == path,
                        MalwareHit.app_name == app_name,
                        MalwareHit.status
                        == MalwareHitStatus.CLEANUP_RESTORE_PENDING,
                    ),
                    MalwareHitStatus.CLEANUP_RESTORE_STARTED,
                )
                _report(operations.mark_running, path, app_name)
            return row

        return await self._launch(await store_db.write(_start))

    async def _launch(self, row) -> bool:
        """Launch *row*'s child, counted before it can run. One that cannot
        be launched finishes its restore, and the caller starts the next."""
        expected = None if row.scan_id is None else (row.scan_id, row.scope)

        def _count_launch():
            return (
                MDSRestoreBatch.update(launches=MDSRestoreBatch.launches + 1)
                .where(
                    MDSRestoreBatch.child_id == row.child_id,
                    MDSRestoreBatch.done == False,  # noqa: E712
                )
                .execute()
            )

        try:
            if not await store_db.write(_count_launch):
                # Settled and removed since the caller read the row.
                logger.info(
                    "MDS restore batch %s is gone, not launching it",
                    row.child_id,
                )
                return True
            await MalwareDatabaseRestore(
                path=row.path,
                app_name=row.app_name,
                restore_id=row.child_id,
                guarded=True,
                expected=expected,
            ).restore()
        except Exception:
            logger.exception(
                "Cannot launch MDS restore batch %s", row.child_id
            )
            await self._finish(row, succeeded=set(), then_next=False)
            return False
        return True

    async def _recover(self, row):
        """*row*'s child has no result: launched again under its id, its
        native journal replays a finished batch and refuses one in doubt.

        Coalesced per child, as messages are handled concurrently: a
        recovery already under way, or a *row* read before its child was
        launched again or finished, launches nothing and removes nothing."""
        if self._stopped or row.child_id in self._relaunching:
            return
        current = MDSRestoreBatch.get_or_none(
            MDSRestoreBatch.child_id == row.child_id
        )
        if current is None or current.done or current.launches != row.launches:
            logger.info(
                "Ignoring a stale recovery of MDS restore batch %s",
                row.child_id,
            )
            return
        row = current
        launched = None  # None: no launch is left
        self._relaunching.add(row.child_id)
        try:
            # A child with no pid file for long may only be slow to start:
            # never run two into the same path, nor fail one that is alive.
            task = asyncio.current_task()
            self._waiting.add(task)
            try:
                alive = await asyncio.get_running_loop().run_in_executor(
                    None, MDSDetachedRestore(row.child_id).has_live_process
                )
            finally:
                self._waiting.discard(task)
            if alive:
                logger.warning(
                    "MDS restore batch %s is still running, not"
                    " relaunching it",
                    row.child_id,
                )
                self._recover_later(row)
                return
            # A completion may have settled the batch while the process
            # table was read: look at the row again before acting on it.
            current = MDSRestoreBatch.get_or_none(
                MDSRestoreBatch.child_id == row.child_id
            )
            if (
                current is None
                or current.done
                or current.launches != row.launches
            ):
                logger.info(
                    "Ignoring a stale recovery of MDS restore batch %s",
                    row.child_id,
                )
                return
            row = current
            if row.launches < self.MAX_BATCH_LAUNCHES:
                logger.warning(
                    "Relaunching MDS restore batch %s", row.child_id
                )
                self._remove_run_dir(row.child_id)
                launched = await self._launch(row)
        finally:
            self._relaunching.discard(row.child_id)
        if launched is None:
            logger.error(
                "MDS restore batch %s has no result after %d launches",
                row.child_id,
                row.launches,
            )
            await self._finish(row, succeeded=set())
        elif not launched:
            await self._restore_next()

    def _recover_later(self, row):
        """Look at *row* again: the checker has called its child lost and
        removed its run directory, so nothing else will come for it."""

        if self._stopped:
            return

        def again():
            self._timers.discard(handle)
            if self._stopped:
                return
            task = asyncio.ensure_future(self._recover(row))
            self._rechecks.add(task)
            task.add_done_callback(self._rechecked)

        handle = asyncio.get_running_loop().call_later(
            self.RECOVERY_RECHECK, again
        )
        self._timers.add(handle)

    def _rechecked(self, task):
        self._rechecks.discard(task)
        if not task.cancelled() and task.exception() is not None:
            logger.error(
                "Looking again at a lost MDS restore batch failed",
                exc_info=task.exception(),
            )

    async def _read_batch(self, row):
        return await MDSDetachedRestore(row.child_id).complete()

    async def _batch_completed(self, row):
        if row.done:
            await self._forget(row.child_id)
            return
        if row.child_id in self._accepting:
            return
        if not MDSDetachedRestoreDir(row.child_id).done_file.is_file():
            # Not finished: an early or a stale delivery.
            return
        self._accepting.add(row.child_id)
        try:
            try:
                result = await self._read_batch(row)
            except ScanAlreadyCompleteError:
                await self._recover(row)
                return
            await self._accept(row, result)
        finally:
            self._accepting.discard(row.child_id)

    async def _accept(self, row, result):
        progress = result.get("restore_batch")
        valid = (
            isinstance(result, MDSDetachedRestore.SUCCESS_MSG)
            and not result["failed"]
            and "restore_batch_error" not in result
        )
        if row.scan_id is not None:
            # A continued batch is bound to the selection it was launched
            # with, and must report on it.
            valid = (
                valid
                and progress is not None
                and (progress.selected_scan_id, progress.scope_key)
                == (row.scan_id, row.scope)
                # Starts where the last batch left off; with each batch
                # restoring something, the count falls and the restore ends.
                and progress.remaining_before == row.remaining
            )
        if (
            valid
            and progress is not None
            and progress.continues
            and result.get("restore_route") == "sqlite"
        ):

            def _continue():
                with MDSRestoreBatch._meta.database.atomic():
                    row.delete_instance()
                    return MDSRestoreBatch.create(
                        child_id=_new_batch_child(),
                        path=row.path,
                        app_name=row.app_name,
                        scan_id=progress.selected_scan_id,
                        scope=progress.scope_key,
                        restored=row.restored
                        + progress.remaining_before
                        - progress.remaining_after,
                        remaining=progress.remaining_after,
                    )

            following = await store_db.write(_continue)
            self._remove_run_dir(row.child_id)
            if not await self._launch(following):
                await self._restore_next()
            return
        # The last batch: an exhausted one on the store route, or the full
        # result of a route without batches (the initial legacy CSV, no
        # backups). A report that names no route, or the store route without
        # its batch object, is no complete result.
        route = result.get("restore_route")
        if route == "sqlite":
            done = (
                valid
                and progress is not None
                and progress.stop_reason == "exhausted"
            )
        else:
            done = (
                valid
                and progress is None
                and row.scan_id is None
                and route in ("csv", "none")
            )
        partial = None
        if not done:
            logger.warning(
                "MDS restore batch %s of %s (%s) failed: %s",
                row.child_id,
                row.path,
                row.app_name,
                self._why_not_done(row, result, valid, progress, route),
            )
            # This batch's own cells count too.
            partial = row.restored + result.get("rows_restored", 0)
        await self._finish(
            row,
            succeeded=result["succeeded"] if done else set(),
            partial=partial,
        )

    @staticmethod
    def _why_not_done(row, result, valid, progress, route):
        """What kept a batch from being the restore's complete result."""
        if not isinstance(result, MDSDetachedRestore.SUCCESS_MSG):
            return "the run failed: {}".format(result.get("error"))
        if "restore_batch_error" in result:
            return "invalid batch: {}".format(result["restore_batch_error"])
        if result["failed"]:
            return "the report has errors"
        if not valid:
            reported = (
                None
                if progress is None
                else (progress.selected_scan_id, progress.scope_key)
            )
            return "continuation expected {}, reported {}".format(
                (row.scan_id, row.scope, row.remaining),
                None
                if reported is None
                else reported + (progress.remaining_before,),
            )
        if route not in ("sqlite", "csv", "none"):
            return "the report names no restore route ({!r})".format(route)
        if route == "sqlite":
            return (
                "the store route reported no batch"
                if progress is None
                else "the batch stopped: {}".format(progress.stop_reason)
            )
        return "the {} route reported a batch or continued one".format(route)

    async def _finish(self, row, succeeded, *, partial=None, then_next=True):
        """Write the status and history of *row*'s hits, once.

        A restore that fails after earlier batches put cells back is
        *partial* (those cells are in the database): its hits stay cleaned,
        as the legacy route leaves them, with a partial-restore event.
        *partial* is the count of cells known put back (earlier batches'
        unless given). *then_next*: start the next restore after it."""
        if partial is None:
            partial = row.restored

        def _settle():
            owned = list(
                MalwareHit.db_hits_under_restoration().where(
                    MalwareHit.orig_file == row.path,
                    MalwareHit.app_name == row.app_name,
                )
            )
            restored_ids = {
                hit.id for hit in MalwareHit.get_db_hits(succeeded)
            }
            restored = [hit for hit in owned if hit.id in restored_ids]
            failed = [hit for hit in owned if hit.id not in restored_ids]
            with MDSRestoreBatch._meta.database.atomic():
                MalwareHit.set_status(restored, MalwareHitStatus.FOUND)
                MalwareHit.set_status(failed, MalwareHitStatus.CLEANUP_DONE)
                _report_outcomes(restored, failed, {(row.path, row.app_name)})
                events = [
                    _restore_event(h, RESTORED_ORIGINAL) for h in restored
                ]
                failure = (
                    PARTIALLY_RESTORED_ORIGINAL
                    if partial
                    else FAILED_TO_RESTORE_ORIGINAL
                )
                events += [_restore_event(h, failure) for h in failed]
                if events:
                    MalwareHistory.save_events(events)
                MDSRestoreBatch.update(done=True).where(
                    MDSRestoreBatch.child_id == row.child_id
                ).execute()
            return failed

        failed = await store_db.write(_settle)
        if failed and partial:
            logger.warning(
                "MDS restore of %s (%s) failed after %d cells were restored",
                row.path,
                row.app_name,
                partial,
            )
        await self._forget(row.child_id)
        if then_next:
            await self._restore_next()

    @staticmethod
    def _remove_run_dir(child_id):
        shutil.rmtree(
            str(MDSDetachedRestoreDir(child_id).path), ignore_errors=True
        )

    async def _forget(self, child_id):
        self._remove_run_dir(child_id)

        def _delete_row():
            MDSRestoreBatch.delete().where(
                MDSRestoreBatch.child_id == child_id
            ).execute()

        await store_db.write(_delete_row)

    async def _reconcile_batches(self):
        """After a restart: finish what was accepted, relaunch a child that
        left no run directory. A child with one is the detached checker's,
        which reports it finished or aborted."""
        for row in list(MDSRestoreBatch.select()):
            if row.done:
                await self._forget(row.child_id)
            elif not MDSDetachedRestoreDir(row.child_id).path.exists():
                await self._recover(row)

    async def create_sink(self, loop):
        self.loop = loop
        self._stopped = False
        await self._reconcile_batches()
        try:
            await store_db.write(operations.interrupt_stale)
        except Exception:
            logger.exception("Cannot update the database restore operations")
        await self._restore_next()

    async def shutdown(self):
        """The agent is deactivated: the timers and the looks they started
        must not outlive the plugin and the database worker. A look cut short
        leaves its row as it is: the next activation recovers it."""
        self._stopped = True
        for handle in self._timers:
            handle.cancel()
        self._timers.clear()
        for task in list(self._rechecks):
            if task in self._waiting:
                await safe_cancel_task(task)
            else:
                # Past the wait it counts a launch and spawns the child, or
                # settles the batch: cut between the two, a launch is used
                # and no child runs. The database worker is still there.
                await asyncio.wait({task}, timeout=self.RECHECK_FINISH_TIMEOUT)

    @staticmethod
    def _batch_owned(hits: Iterable[MalwareHit]) -> List[MalwareHit]:
        """*hits* less those of an application restore in batches: only its
        own batches decide their status and history."""
        active = {
            (row.path, row.app_name)
            for row in MDSRestoreBatch.select().where(
                MDSRestoreBatch.done == False  # noqa: E712
            )
        }
        return [
            hit for hit in hits if (hit.orig_file, hit.app_name) not in active
        ]

    @staticmethod
    def _still_started(hits: Iterable[MalwareHit]) -> List[MalwareHit]:
        return [
            hit
            for hit in hits
            if hit.status == MalwareHitStatus.CLEANUP_RESTORE_STARTED
        ]

    @expect(MessageType.MalwareDatabaseRestoreTask)
    async def queue_db_restore(self, message: MalwareDatabaseRestoreTask):
        # One step with the start of a batch, which claims the hits: the
        # check for its row and the update of the hits must not straddle
        # the start's commit.
        async with self._starting:
            query = (
                MalwareHit.db_hits()
                .where(MalwareHit.orig_file == message.path)
                .where(MalwareHit.app_name == message.app_name)
            )
            if (
                MDSRestoreBatch.select()
                .where(
                    MDSRestoreBatch.path == message.path,
                    MDSRestoreBatch.app_name == message.app_name,
                    MDSRestoreBatch.done == False,  # noqa: E712
                )
                .exists()
            ):
                # The hits of a restore in batches stay with it; a new request
                # queues the others, to run after it. Any other started hit is
                # requeued, as before batches: one left without a child must not
                # hold every restore for good.
                query = query.where(
                    MalwareHit.status
                    != MalwareHitStatus.CLEANUP_RESTORE_STARTED
                )
            restore_status = MalwareHitStatus.CLEANUP_RESTORE_PENDING
            if message.signature_id:
                query = query.where(MalwareHit.type == message.signature_id)
                restore_status = (
                    MalwareHitStatus.CLEANUP_REMOTE_RESTORE_PENDING
                )

            MalwareHit.set_status(
                query,
                restore_status,
            )
        await self._restore_next()

    @expect(MessageType.MalwareRestoreComplete)
    async def parse_restore_results(self, message):
        restore_id = message["scan_id"]
        if (
            row := MDSRestoreBatch.get_or_none(
                MDSRestoreBatch.child_id == restore_id
            )
        ) is not None:
            await self._batch_completed(row)
            return
        if _is_batch_child(restore_id):
            # A batch child no row has any more (finished, or a run left
            # behind): its result is no other restore's.
            logger.info(
                "Ignoring a late completion of MDS restore batch %s",
                restore_id,
            )
            self._remove_run_dir(restore_id)
            return
        if not self._completions.take(restore_id):
            logger.info(
                "Ignoring a duplicate completion of restore %s", restore_id
            )
            return
        detached_restore = MDSDetachedRestore(restore_id)

        try:
            restore_message = await detached_restore.complete()
        except ScanAlreadyCompleteError:
            # This happens when AV is woken up by AiBolit. See DEF-11078.
            logger.warning(
                "Cannot complete restore %s, assuming it is already complete",
                restore_id,
            )
            return
        finally:
            shutil.rmtree(
                str(detached_restore.detached_dir), ignore_errors=True
            )

        await g.sink.process_message(restore_message)

    @expect(MessageType.MalwareDatabaseRestore)
    async def update_restored_hits_status(self, message):
        restored_hits = self._batch_owned(
            MalwareHit.get_db_hits(message.succeeded)
        )
        restored_now = self._still_started(restored_hits)
        MalwareHit.set_status(restored_now, MalwareHitStatus.FOUND)
        # Failed hits leave the under-restoration status here so the
        # leftover sweep does not report them a second time; their
        # history event comes from save_restore_events_in_history.
        failed_hits = self._batch_owned(MalwareHit.get_db_hits(message.failed))
        failed_now = self._still_started(failed_hits)
        MalwareHit.set_status(failed_now, MalwareHitStatus.CLEANUP_DONE)
        _report_outcomes(
            restored_now,
            failed_now,
            {
                (info.path, info.app_name)
                for info in message.succeeded | message.failed
            },
        )
        # The queue is serial, so any hit still under restoration that no
        # other run directory owns belongs to this operation (whose own
        # directory is gone by now); without a terminal status it would
        # block the queue forever. Not scoped to the report for the same
        # reason as the cleanup sweep: a run that fails before reaching any
        # database reports no hits at all.
        under_restoration = MalwareHit.db_hits_under_restoration()
        if leftover_hits := self._batch_owned(under_restoration):
            MalwareHit.set_status(leftover_hits, MalwareHitStatus.CLEANUP_DONE)
            _report_outcomes([], leftover_hits)
            for hit in leftover_hits:
                await MalwareAction.cleanup_failed_restore(
                    path=hit.orig_file,
                    app_name=hit.app_name,
                    resource_type=MalwareScanResourceType.DB.value,
                    file_owner=hit.owner,
                    file_user=hit.user,
                    initiator=message.get("initiator"),
                    cause=message.get("cause"),
                    db_host=hit.db_host,
                    db_port=hit.db_port,
                    db_name=hit.db_name,
                    signature_id=hit.signature_id,
                )
        await self._restore_next()

    @expect(MessageType.MalwareDatabaseRestore)
    async def save_restore_events_in_history(self, message):
        cause = message.get("cause")
        initiator = message.get("initiator")
        restored_hits = self._batch_owned(
            MalwareHit.get_db_hits(message.succeeded)
        )
        for hit in restored_hits:
            # FIXME: change cleanup_restored_original to accept multiple
            # values
            await MalwareAction.cleanup_restored_original(
                path=hit.orig_file,
                app_name=hit.app_name,
                resource_type=MalwareScanResourceType.DB.value,
                file_owner=hit.owner,
                file_user=hit.user,
                initiator=initiator,
                cause=cause,
                db_host=hit.db_host,
                db_port=hit.db_port,
                db_name=hit.db_name,
                signature_id=hit.signature_id,
            )
        failed_hits = self._batch_owned(MalwareHit.get_db_hits(message.failed))
        for hit in failed_hits:
            await MalwareAction.cleanup_failed_restore(
                path=hit.orig_file,
                app_name=hit.app_name,
                resource_type=MalwareScanResourceType.DB.value,
                file_owner=hit.owner,
                file_user=hit.user,
                initiator=initiator,
                cause=cause,
                db_host=hit.db_host,
                db_port=hit.db_port,
                db_name=hit.db_name,
                signature_id=hit.signature_id,
            )

    @expect(MessageType.MalwareDatabaseRestoreFailed)
    async def update_failed_hits_status(self, message):
        """
        Clear the queue when the restore fails,
        set hits' status back to cleanup_done
        """
        child = message.get("scan_id")
        if _is_batch_child(child) and not MDSRestoreBatch.get_or_none(
            MDSRestoreBatch.child_id == child
        ):
            # A batch child no row has any more: its restore is over, and its
            # failure is no other restore's.
            logger.info(
                "Ignoring a late failure of MDS restore batch %s", child
            )
            return
        if child and (
            row := MDSRestoreBatch.get_or_none(
                MDSRestoreBatch.child_id == child
            )
        ):
            # A batch child found dead: its directory is gone, its outcome is
            # the native journal's to tell.
            if row.done:
                return
            # The abort handler removes the directory before it reports, so
            # a directory there, or a relaunch under way, means this failure
            # is about the launch already replaced (two checks of one dead
            # child report it twice). A completion being accepted means the
            # child finished as the checker looked: its result decides.
            if (
                row.child_id in self._relaunching
                or row.child_id in self._accepting
                or MDSDetachedRestoreDir(row.child_id).path.exists()
            ):
                logger.info(
                    "Ignoring a stale failure of MDS restore batch %s", child
                )
                return
            await self._recover(row)
            return
        hits = self._batch_owned(MalwareHit.db_hits_under_restoration())
        MalwareHit.set_status(hits, MalwareHitStatus.CLEANUP_DONE)
        _report_outcomes([], hits)
        for hit in hits:
            await MalwareAction.cleanup_failed_restore(
                path=hit.orig_file,
                app_name=hit.app_name,
                resource_type=MalwareScanResourceType.DB.value,
                file_owner=hit.owner,
                file_user=hit.user,
                initiator=message.get("initiator"),
                cause=message.get("cause"),
                db_host=hit.db_host,
                db_port=hit.db_port,
                db_name=hit.db_name,
                signature_id=hit.signature_id,
            )
        await self._restore_next()
