#!/usr/bin/env python3
#
# cook-full - recook the whole wok in a dedicated build environment.
#
# Order engine: reads the wok and computes the build order (arch-correct:
# receipts are *sourced* with ARCH, so conditional BUILD_DEPENDS and SPLIT
# are honoured). Receipts v2 produce several packages from one receipt, so a
# bdep is resolved to the receipt that produces it, and followed through its
# DEPENDS (read from the seed) because cook installs that whole tree.
#
# Build driver: walks the order calling `cook <pkg>`, tracks state (sqlite if
# python3 has it, else a TSV fallback) and resumes. The environment's $PKGS
# starts as a copy of the seed, a snapshot of the production packages, and
# each cook replaces its own: a failure blocks nothing, so one pass finds
# nearly every broken receipt. Only a broken environment stops the run.
# Source doctor: tarballs are magic-checked, stub/dead sources are seeded
# from mirror.slitaz.org/sources and re-cooked, and a background thread
# prefetches upcoming tarballs while the current package compiles.
#
# --dry-run / --status / --simulate cook NOTHING. A real run refuses to start
# without a seed, which keeps it out of the production cooker.
#
# Copyright (C) SliTaz GNU/Linux - GNU GPL v3

import argparse
import fnmatch
import glob
import hashlib
import html
import json
import os
import shutil
import subprocess
import sys
import tempfile
import time

try:
    import sqlite3
    HAVE_SQLITE = True
except ImportError:
    HAVE_SQLITE = False

# Shell scanner: source each receipt (wok dir names on stdin) in a subshell
# with ARCH exported, emit one TSV row per package. $(echo $VAR) collapses
# newlines/tabs/runs of spaces in the multi-line dep vars to single spaces.
SCAN = r'''
# mirror variables receipts expand in WGET_URL ($GNU_MIRROR, $SF_MIRROR...);
# sourced FIRST: cook.conf also sets ARCH/WOK and must not override the args
. /etc/slitaz/cook.conf 2>/dev/null
ARCH="$1"; WOK="$2"; export ARCH
while read d; do
	r="$WOK/$d/receipt"
	[ -s "$r" ] || continue
	(
		PACKAGE=; HOST_ARCH=; WANTED=; BUILD_DEPENDS=; DEPENDS=
		TARBALL=; WGET_URL=; SPLIT=
		. "$r" 2>/dev/null
		printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
			"${PACKAGE:-$d}" "$d" "$(echo $HOST_ARCH)" \
			"$(echo $WANTED)" "$(echo $BUILD_DEPENDS)" "$(echo $DEPENDS)" \
			"$(echo $TARBALL)" "$(echo $WGET_URL)" "$(echo $SPLIT)"
	) </dev/null
done
'''

# bump when SCAN's output or the cache row format changes
SCAN_CACHE_V = '#cook-full-scan-v3'

# Where a recook environment keeps its snapshot of the production packages
DEFAULT_SEED = '/home/slitaz/packages.seed'


def conf(name, default=None):
    """Read a variable from /etc/slitaz/cook.conf (best effort)."""
    path = '/etc/slitaz/cook.conf'
    if not os.path.exists(path):
        return default
    try:
        out = subprocess.run(
            ['sh', '-c', '. "$1" 2>/dev/null; eval "echo \\$$2"',
             '_', path, name],
            capture_output=True, text=True, timeout=10).stdout.strip()
        return out or default
    except Exception:
        return default


def host_arch():
    m = os.uname().machine
    if m == 'x86_64':
        return 'x86_64'
    return 'i486'  # i?86 and friends


class Pkg:
    __slots__ = ('name', 'wokdir', 'host_arch', 'wanted', 'bdeps', 'deps',
                 'tarball', 'wget', 'split')

    def __init__(self, name, wokdir, host_arch, wanted, bdeps, deps,
                 tarball='', wget='', split=()):
        self.name = name
        self.wokdir = wokdir
        self.host_arch = host_arch
        self.wanted = wanted        # list (receipts v1, usually empty)
        self.bdeps = bdeps          # list
        self.deps = deps            # list (runtime; for info, not ordering)
        self.tarball = tarball      # source tarball file name ('' if none)
        self.wget = wget            # upstream WGET_URL ('' if none)
        # receipts v2: SPLIT is 'pkg1 pkg2:set1 pkg3:set2'
        self.split = [s.split(':')[0] for s in split]

    def outputs(self):
        """Package names this receipt produces."""
        return [self.name] + [s for s in self.split if s != self.name]


def scan_wok(wok, arch, cachefile=None):
    """Return {name: Pkg} by sourcing receipts with ARCH set.

    With cachefile, rows are cached keyed on (arch, receipt mtime): only
    receipts that changed since the last scan are re-sourced, so the
    typical fix-and-`--continue` cycle restarts in ~1s instead of
    re-forking ~6k subshells."""
    # wokdir -> [mtime, name, ha, wanted, bdeps, deps, tarball, wget, split]
    entries = {}
    if cachefile:
        try:
            with open(cachefile) as fh:
                # versioned header: a SCAN/format change must drop old rows
                # even though the receipts' mtimes did not move
                if fh.readline().rstrip('\n') == SCAN_CACHE_V:
                    for line in fh:
                        f = line.rstrip('\n').split('\t')
                        if len(f) == 11 and f[0] == arch:
                            entries[f[1]] = f[2:]
        except OSError:
            pass

    seen = set()
    todo = []           # (wokdir, mtime) to (re-)source
    for d in sorted(os.listdir(wok)):
        try:
            mt = '%d' % os.stat(os.path.join(wok, d, 'receipt')).st_mtime
        except OSError:
            continue
        seen.add(d)
        e = entries.get(d)
        if not e or e[0] != mt:
            todo.append((d, mt))
    pruned = set(entries) - seen        # receipt deleted from the wok
    for d in pruned:
        del entries[d]

    if todo:
        mtimes = dict(todo)
        res = subprocess.run(['sh', '-c', SCAN, 'cook-full-scan', arch, wok],
                             input='\n'.join(d for d, _ in todo) + '\n',
                             capture_output=True, text=True)
        for line in res.stdout.splitlines():
            f = line.split('\t')
            if len(f) != 9 or not f[0] or f[1] not in mtimes:
                continue
            entries[f[1]] = [mtimes[f[1]], f[0], f[2], f[3], f[4], f[5],
                             f[6], f[7], f[8]]

    if cachefile and (todo or pruned):
        tmp = cachefile + '.tmp'
        try:
            with open(tmp, 'w') as fh:
                fh.write(SCAN_CACHE_V + '\n')
                for d in sorted(entries):
                    fh.write('%s\t%s\t%s\n' % (arch, d, '\t'.join(entries[d])))
            os.replace(tmp, cachefile)
        except OSError:
            pass

    pkgs = {}
    for wokdir, e in entries.items():
        _, name, ha, wanted, bdeps, deps, tarball, wget, split = e
        pkgs[name] = Pkg(name, wokdir, ha.split(),
                         wanted.split(), bdeps.split(), deps.split(),
                         tarball, wget, split.split())
    return pkgs


def arch_ok(pkg, arch):
    """arch_db rule: unset HOST_ARCH = current arch; else must list arch|any."""
    if not pkg.host_arch:
        return True
    return arch in pkg.host_arch or 'any' in pkg.host_arch


class Seed:
    """Read-only snapshot of the production packages, taken when the recook
    environment is created. $PKGS starts as a copy of it and each cook replaces
    its packages, so a bdep is always installable, whatever the build order and
    whatever failed. It also holds what a receipt v2 can't tell without running
    its genpkg_rules: the DEPENDS of each split, which cook's bdep install pulls
    in, and the PROVIDE aliases."""

    def __init__(self, path):
        self.path = path
        self.present = False
        self.rows = {}          # name -> packages.info line
        self.depends = {}       # name -> [runtime deps]
        self.provides = {}      # alias -> name
        try:
            with open(os.path.join(path, 'packages.info'),
                      errors='replace') as fh:
                for line in fh:
                    f = line.rstrip('\n').split('\t')
                    if len(f) < 10:
                        continue
                    self.rows[f[0]] = line
                    self.depends[f[0]] = f[7].split()
                    for alias in f[9].split():
                        self.provides.setdefault(alias.split(':')[0], f[0])
            self.present = bool(self.rows)
        except OSError:
            pass

    def restore(self, names, pkgsdir):
        """Bring back the named packages $PKGS lost, with their index rows.
        Returns the restored names; a package the index still lists and whose
        file exists is left alone, whatever its version."""
        info = os.path.join(pkgsdir, 'packages.info')
        try:
            with open(info, errors='replace') as fh:
                current = {l.split('\t', 1)[0]: l.split('\t')[1] for l in fh
                           if '\t' in l}
        except OSError:
            current = {}
        restored = []
        for name in names:
            line = self.rows.get(name)
            if line is None:
                continue
            ver = current.get(name)
            if ver and tazpkg_file(pkgsdir, name, ver):
                continue
            ver = line.split('\t')[1]
            src = tazpkg_file(self.path, name, ver)
            if not src:
                continue
            base = os.path.basename(src)
            dst = os.path.join(pkgsdir, base)
            # Never write through an existing file: $PKGS holds hard links, a
            # copy over one would rewrite the package on the other side too.
            if not os.path.exists(dst):
                shutil.copy2(src, dst + '.tmp')
                os.replace(dst + '.tmp', dst)
            if name not in current:
                with open(info, 'a') as fh:
                    fh.write(line)
            with open(src, 'rb') as fh:
                md5 = hashlib.md5(fh.read()).hexdigest()
            append_once(os.path.join(pkgsdir, 'packages.md5'),
                        '%s  %s\n' % (md5, base), '  %s\n' % base)
            append_once(os.path.join(pkgsdir, 'packages.list'),
                        base[:-len('.tazpkg')] + '\n',
                        '\n' + base[:-len('.tazpkg')] + '\n')
            restored.append(name)
        return restored


def append_once(path, line, marker):
    """Append line unless marker is already in the index file."""
    try:
        with open(path, errors='replace') as fh:
            if marker in '\n' + fh.read():
                return
    except OSError:
        pass
    with open(path, 'a') as fh:
        fh.write(line)


def tazpkg_file(pkgsdir, name, ver):
    """Path of name-ver[-arch].tazpkg ('' if absent): i486 packages have no
    arch suffix, the others carry -$ARCH or -any."""
    plain = os.path.join(pkgsdir, '%s-%s.tazpkg' % (name, ver))
    if os.path.exists(plain):
        return plain
    found = glob.glob(os.path.join(glob.escape(pkgsdir),
                                   glob.escape('%s-%s-' % (name, ver))
                                   + '*.tazpkg'))
    return found[0] if found else ''


def read_installed(root):
    try:
        return set(os.listdir(os.path.join(root, 'var/lib/tazpkg/installed')))
    except OSError:
        return set()


def mirror_is_local(root):
    """Same rule as cook: the first non-blank line of the mirror file."""
    try:
        with open(os.path.join(root, 'var/lib/tazpkg/mirror')) as fh:
            for line in fh:
                if line.strip():
                    return line.strip().startswith('/')
    except OSError:
        pass
    return False


def build_need(pkgs, arch, installed, seed):
    """{package: receipts to rebuild first}: what cook installs for it, i.e.
    its BUILD_DEPENDS (and the implicit foo of a foo-dev) with their whole
    DEPENDS tree, mapped from split names to the receipt producing them. What
    the base already has stops the walk: cook won't install it.

    Also returns the direct bdeps nothing provides ({dep: [users]}), which make
    cook die with 'unknown dep', and the seed packages no receipt produces."""
    universe = {n for n, p in pkgs.items() if arch_ok(p, arch)}
    owner = {}
    for n in universe:
        for out in pkgs[n].outputs():
            owner.setdefault(out, n)
    for n in universe:          # a receipt always owns its own name
        owner[n] = n

    def real(x):
        return x if x in owner or x in seed.depends else seed.provides.get(x)

    need = {}
    unknown = {}
    for n in universe:
        p = pkgs[n]
        for d in p.bdeps + p.wanted:
            if d not in installed and real(d) is None:
                unknown.setdefault(d, []).append(n)
        stack = p.bdeps + p.wanted + [b[:-4] for b in p.bdeps
                                      if b.endswith('-dev')]
        seen = set()
        while stack:
            x = real(stack.pop())
            if x is None or x in seen or x in installed:
                continue
            seen.add(x)
            stack.extend(seed.depends.get(x, ()))
        need[n] = {owner[x] for x in seen if x in owner} - {n}
    orphans = sorted(set(seed.rows) - set(owner))
    return need, unknown, orphans


def compute_order(need):
    """Dependency order, in waves: most used packages first within a wave.
    The wok has dependency cycles once DEPENDS count; a package in one takes
    the seed's copy of what isn't rebuilt yet, so a cycle is cut, never fatal.
    Returns (order, [(package, deps not rebuilt yet)])."""
    rdeg = {n: 0 for n in need}
    for n in need:
        for d in need[n]:
            rdeg[d] += 1
    done = set()
    order = []
    breaks = []
    remaining = set(need)
    while remaining:
        ready = [n for n in remaining if need[n] <= done]
        if ready:
            ready.sort(key=lambda n: (-rdeg[n], n))
        else:
            # least unmet deps for the most users: cuts the fewest edges
            pick = min(remaining,
                       key=lambda n: (len(need[n] - done) - rdeg[n], n))
            breaks.append((pick, sorted(need[pick] - done)))
            ready = [pick]
        order.extend(ready)
        done.update(ready)
        remaining.difference_update(ready)
    return order, breaks


def dependents(need):
    """{package: how many packages build against it} (need is transitive)."""
    count = {}
    for n in need:
        for d in need[n]:
            count[d] = count.get(d, 0) + 1
    return count


# ---------------------------------------------------------------------------
# M2: state store (sqlite if available, TSV fallback) + build driver
# ---------------------------------------------------------------------------

STATUSES = ('todo', 'building', 'done', 'failed', 'skipped')


def now():
    return time.strftime('%Y-%m-%d %H:%M:%S')


def broken_set(cache):
    try:
        with open(os.path.join(cache, 'broken')) as fh:
            return set(l.strip() for l in fh if l.strip())
    except OSError:
        return set()



def run_cook(name, logpath):
    """Call `cook <pkg>`, appending its output to logpath. Returns exit code.
    Only ever called on a real run (never by the agent / --simulate)."""
    with open(logpath, 'a') as fh:
        fh.write('\n===== cook %s  (%s) =====\n' % (name, now()))
        fh.flush()
        try:
            return subprocess.call(['cook', name], stdout=fh,
                                   stderr=subprocess.STDOUT)
        except FileNotFoundError:
            fh.write('cook not found in PATH\n')
            return 127


def toolchain_canary():
    """Sanity-check the build environment BEFORE cooking anything: the C
    compiler must produce a runnable executable and `ar` must be GNU ar.
    Seen in production (2026-06-11): a busybox rebuild swept the binutils
    /usr/bin symlinks (as, ar, nm, ...) -- every subsequent package failed
    with 'C compiler cannot create executables' / 'could not determine ar
    interface' until someone noticed. Cheap to catch up front.
    Returns a list of problem strings (empty = healthy)."""
    probs = []
    with tempfile.TemporaryDirectory() as td:
        cfile = os.path.join(td, 'canary.c')
        exe = os.path.join(td, 'canary')
        with open(cfile, 'w') as fh:
            # <linux/limits.h> on purpose: a removed kernel build-dep can
            # sweep /usr/include/linux/* that linux-api-headers still owns
            # (tazpkg remove doesn't ref-count shared files) -> cpp sanity
            # fails for every subsequent ./configure. A bare `int main` would
            # not notice; including a kernel header makes the canary trip.
            fh.write('#include <stdio.h>\n'
                     '#include <linux/limits.h>\n'
                     'int main(void){return PATH_MAX > 0 ? 0 : 1;}\n')
        r = None
        for cc in ('cc', 'gcc'):
            try:
                r = subprocess.run([cc, cfile, '-o', exe],
                                   capture_output=True, text=True, timeout=60)
                break
            except (FileNotFoundError, subprocess.TimeoutExpired):
                continue
        if r is None:
            probs.append('no cc/gcc in PATH')
        elif r.returncode != 0:
            err = r.stderr.strip().splitlines()
            probs.append('cc cannot build an executable (%s)'
                         % (err[-1] if err else 'no diagnostic'))
        else:
            try:
                if subprocess.run([exe], timeout=10).returncode != 0:
                    probs.append('canary executable does not run')
            except (OSError, subprocess.TimeoutExpired):
                probs.append('canary executable does not run')
    try:
        out = subprocess.run(['ar', '--version'], capture_output=True,
                             text=True, timeout=10).stdout
        first = out.splitlines()[0] if out else ''
        if 'GNU ar' not in first:
            probs.append('ar is not GNU ar (binutils symlink swept?)')
    except (FileNotFoundError, subprocess.TimeoutExpired):
        probs.append('no ar in PATH')
    # python3 must be runnable: SliTaz has two python3 stacks (python3 vs the
    # legacy py3k) fighting over /usr/bin/python3; a py3k build-dep removal
    # sweeps the symlink (tazpkg remove doesn't restore the modified file),
    # then any python3-shebang build tool (gtkdoc-rebase, ...) dies with a
    # misleading '/bin/sh: ...: not found' pointing at the script not python3.
    try:
        if subprocess.run(['python3', '--version'], capture_output=True,
                          timeout=10).returncode != 0:
            probs.append('python3 missing or broken')
    except (FileNotFoundError, subprocess.TimeoutExpired):
        probs.append('python3 missing or broken')
    return probs



def load_skip(path, extra=''):
    """Glob patterns of packages to intentionally skip (data/skip + --skip)."""
    pats = set(p for p in extra.split(',') if p)
    if path and os.path.exists(path):
        with open(path) as fh:
            pats.update(l.strip() for l in fh
                        if l.strip() and not l.startswith('#'))
    return pats


def load_triage(path):
    """Ordered [(glob, action, category, note)] from data/triage.tsv -- the
    curated human narrative (fix idea / why we drop) the db can't carry."""
    rows = []
    if path and os.path.exists(path):
        with open(path) as fh:
            for l in fh:
                if not l.strip() or l.lstrip().startswith('#'):
                    continue
                f = l.rstrip('\n').split('\t')
                if len(f) >= 4:
                    rows.append((f[0], f[1], f[2], f[3]))
    return rows


def triage_for(name, triage):
    """First matching triage row (glob, fnmatch) -> (action, cat, note)."""
    for glob, action, cat, note in triage:
        if fnmatch.fnmatch(name, glob):
            return action, cat, note
    return None


class SqliteStore:
    def __init__(self, path):
        self.db = sqlite3.connect(path)
        self.db.execute(
            'CREATE TABLE IF NOT EXISTS pkg('
            'name TEXT PRIMARY KEY, pos INTEGER DEFAULT -1, '
            'status TEXT DEFAULT "todo", seconds REAL DEFAULT 0, '
            'attempts INTEGER DEFAULT 0, culprit TEXT DEFAULT "", '
            'updated TEXT DEFAULT "")')
        self.db.commit()

    backend = 'sqlite'

    def sync_order(self, order):
        rows = [(name, i) for i, name in enumerate(order)]
        self.db.executemany('INSERT OR IGNORE INTO pkg(name) VALUES(?)',
                            [(n,) for n, _ in rows])
        self.db.executemany('UPDATE pkg SET pos=? WHERE name=?',
                            [(i, n) for n, i in rows])
        self.db.commit()

    def get(self, name):
        r = self.db.execute(
            'SELECT status,seconds,attempts,culprit,pos FROM pkg WHERE name=?',
            (name,)).fetchone()
        if not r:
            return {'status': 'unknown', 'seconds': 0, 'attempts': 0,
                    'culprit': '', 'pos': -1}
        return {'status': r[0], 'seconds': r[1], 'attempts': r[2],
                'culprit': r[3], 'pos': r[4]}

    def set(self, name, status, seconds=None, attempts=None, culprit=None):
        self.db.execute('INSERT OR IGNORE INTO pkg(name) VALUES(?)', (name,))
        sets = ['status=?', 'updated=?']
        vals = [status, now()]
        if seconds is not None:
            sets.append('seconds=?'); vals.append(seconds)
        if attempts is not None:
            sets.append('attempts=?'); vals.append(attempts)
        if culprit is not None:
            sets.append('culprit=?'); vals.append(culprit)
        vals.append(name)
        self.db.execute('UPDATE pkg SET %s WHERE name=?' % ','.join(sets), vals)
        self.db.commit()

    def counts(self):
        c = {s: 0 for s in STATUSES}
        for status, n in self.db.execute(
                'SELECT status,COUNT(*) FROM pkg GROUP BY status'):
            c[status] = n
        return c

    def by_status(self, status, limit=None):
        q = 'SELECT name FROM pkg WHERE status=? ORDER BY pos'
        if limit:
            q += ' LIMIT %d' % limit
        return [r[0] for r in self.db.execute(q, (status,))]

    def statuses(self):
        """One-query {name: status} snapshot (drive keeps it as a mirror)."""
        return dict(self.db.execute('SELECT name,status FROM pkg'))

    def flush(self):
        pass

    def close(self):
        self.db.close()


class TsvStore:
    """Flat-file fallback when python3 has no sqlite3 (e.g. x86_64 port).
    Writes are deferred (dirty flag): the full file is only rewritten by
    flush() -- the driver flushes with each progress snapshot and on exit,
    so a crash loses at most ~2s of state ('building' re-cooks on resume)."""
    backend = 'tsv'

    def __init__(self, path):
        self.path = path
        self.d = {}
        self.dirty = False
        try:
            with open(path) as fh:
                for line in fh:
                    f = line.rstrip('\n').split('\t')
                    if len(f) == 6:
                        self.d[f[0]] = {'pos': int(f[1]), 'status': f[2],
                                        'seconds': float(f[3]),
                                        'attempts': int(f[4]), 'culprit': f[5]}
        except OSError:
            pass

    def flush(self):
        if not self.dirty:
            return
        tmp = self.path + '.tmp'
        with open(tmp, 'w') as fh:
            for name in sorted(self.d, key=lambda n: self.d[n]['pos']):
                r = self.d[name]
                fh.write('%s\t%d\t%s\t%g\t%d\t%s\n' % (
                    name, r['pos'], r['status'], r['seconds'],
                    r['attempts'], r['culprit']))
        os.replace(tmp, self.path)
        self.dirty = False

    def sync_order(self, order):
        for i, name in enumerate(order):
            r = self.d.setdefault(name, {'pos': i, 'status': 'todo',
                                         'seconds': 0, 'attempts': 0,
                                         'culprit': ''})
            r['pos'] = i
        self.dirty = True
        self.flush()

    def get(self, name):
        return dict(self.d.get(name, {'status': 'unknown', 'seconds': 0,
                                      'attempts': 0, 'culprit': '', 'pos': -1}))

    def set(self, name, status, seconds=None, attempts=None, culprit=None):
        r = self.d.setdefault(name, {'pos': -1, 'status': 'todo', 'seconds': 0,
                                     'attempts': 0, 'culprit': ''})
        r['status'] = status
        if seconds is not None:
            r['seconds'] = seconds
        if attempts is not None:
            r['attempts'] = attempts
        if culprit is not None:
            r['culprit'] = culprit
        self.dirty = True

    def counts(self):
        c = {s: 0 for s in STATUSES}
        for r in self.d.values():
            c[r['status']] = c.get(r['status'], 0) + 1
        return c

    def by_status(self, status, limit=None):
        names = sorted((n for n, r in self.d.items() if r['status'] == status),
                       key=lambda n: self.d[n]['pos'])
        return names[:limit] if limit else names

    def statuses(self):
        return {n: r['status'] for n, r in self.d.items()}

    def close(self):
        self.flush()


def open_store(sdir, backend):
    use_sqlite = HAVE_SQLITE if backend == 'auto' else (backend == 'sqlite')
    if use_sqlite:
        return SqliteStore(os.path.join(sdir, 'state.db'))
    return TsvStore(os.path.join(sdir, 'state.tsv'))


# Failure log patterns that mean "try again" rather than "broken" -- network
# / fetch hiccups, not a real build error.
TRANSIENT = ('could not resolve', 'unable to resolve', 'temporary failure',
             'connection timed out', 'connection refused', 'network is '
             'unreachable', 'failed to connect', 'timed out', 'download '
             'failed', 'cannot download', 'name or service not known',
             'wget: error', 'tls handshake')


def tail_text(path, lines=80, maxbytes=65536):
    """Last `lines` lines of a (possibly huge) file, read from the end only:
    run.log grows to hundreds of MB over a full build, so never slurp it."""
    try:
        with open(path, 'rb') as fh:
            fh.seek(0, 2)
            size = fh.tell()
            fh.seek(max(0, size - maxbytes))
            data = fh.read()
    except OSError:
        return ''
    return '\n'.join(data.decode('utf-8', 'replace').splitlines()[-lines:])


def last_cook_text(logpath, lines=80):
    """Log tail cut at the last cook's header (see run_cook): run.log is shared,
    and the network error of the package before isn't this package's."""
    return tail_text(logpath, lines).rsplit('===== cook ', 1)[-1].lower()


def is_transient(logpath, lines=80):
    blob = last_cook_text(logpath, lines)
    return any(p in blob for p in TRANSIENT)


# Source-download failure: cook tried WGET_URL *and* its $MIRROR_URL fallback
# (and wget retried internally) and still couldn't fetch the tarball. This is a
# dead/unreachable source URL, not a build bug and not worth re-cooking -- we
# record it as its own category ('src-download') instead of a generic failure.
# NB: no DNS patterns here -- those are TRANSIENT (whole network blinking, not
# a dead URL) and the driver checks transient first / retries before this.
DL_FAIL = ('error: wget ', "can't connect to remote host",
           '404 not found', '403 forbidden')


def is_dl_fail(logpath, lines=80):
    blob = last_cook_text(logpath, lines)
    return any(p in blob for p in DL_FAIL)


# ---------------------------------------------------------------------------
# Source doctor: verify tarballs, rescue from the SliTaz sources mirror,
# prefetch upcoming ones in the background.
#
# The single biggest halt cause in the i486 full run was the "stub source":
# a dead upstream URL serving an HTML error page that wget happily saves as
# the tarball -- extraction then fails (dconf, audit, the 13-member
# uclibc-cross-compiler family...). Every fix was the same manual dance:
# delete the stub, fetch the real tarball from
# mirror.slitaz.org/sources/packages/<initial>/, re-cook. Automate exactly
# that dance.
# ---------------------------------------------------------------------------

SOURCES_MIRROR = 'http://mirror.slitaz.org/sources/packages'

SRC_MAGIC = (
    (b'\x1f\x8b', ('.tar.gz', '.tgz', '.gz')),
    (b'BZh', ('.tar.bz2', '.tbz', '.tbz2', '.bz2')),
    (b'\xfd7zXZ\x00', ('.tar.xz', '.txz', '.xz')),
    (b']\x00\x00', ('.tar.lzma', '.lzma')),
    (b'PK\x03\x04', ('.zip',)),
    (b"7z\xbc\xaf", ('.7z',)),
)


def tarball_ok(path):
    """Is the saved tarball plausibly real? False = stub/corrupt (a dead
    URL serving an error page that wget 'saved' with exit code 0)."""
    try:
        with open(path, 'rb') as fh:
            head = fh.read(512)
    except OSError:
        return False
    if not head:
        return False
    low = path.lower()
    for magic, exts in SRC_MAGIC:
        if low.endswith(exts):
            return head.startswith(magic)
    if low.endswith('.tar'):
        try:
            with open(path, 'rb') as fh:
                fh.seek(257)
                return fh.read(5) == b'ustar'
        except OSError:
            return False
    # unknown extension: at least call out the obvious HTML error page
    sniff = head[:256].lstrip().lower()
    return not (sniff.startswith(b'<!doctype') or sniff.startswith(b'<html'))


def fetch_url(url, dest, timeout=600):
    """wget url -> dest, atomically and verified: True only if the result
    looks like a real tarball (magic bytes), never leaves a partial file."""
    tmp = dest + '.part'
    try:
        rc = subprocess.call(['wget', '-q', '-T', '60', '-O', tmp, url],
                             stdout=subprocess.DEVNULL,
                             stderr=subprocess.DEVNULL, timeout=timeout)
    except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
        rc = 1
    if rc == 0 and tarball_ok(tmp):
        os.replace(tmp, dest)
        return True
    try:
        os.remove(tmp)
    except OSError:
        pass
    return False


def mirror_url(tarball):
    return '%s/%s/%s' % (SOURCES_MIRROR, tarball[0].lower(), tarball)


def is_vcs_url(url):
    return url.split('|')[0] in ('git', 'hg', 'svn', 'bzr', 'cvs')


def rescue_source(pkg, srcdir, msg):
    """After a failed cook: if the source tarball is missing or a stub,
    seed the real one from the SliTaz sources mirror. Returns True when a
    fresh tarball was put in place (caller re-cooks)."""
    if not pkg.tarball:
        return False
    path = os.path.join(srcdir, pkg.tarball)
    if os.path.exists(path):
        if tarball_ok(path):
            return False        # source looks sane: a real build failure
        msg('rescue %s: %s is a stub/corrupt file, trying the mirror\n'
            % (pkg.name, pkg.tarball))
    # $SRC is shared with the production cooker: a bad file is only ever
    # replaced by a good download (fetch_url is atomic), never deleted first.
    if fetch_url(mirror_url(pkg.tarball), path):
        msg('rescue %s: %s seeded from the sources mirror\n'
            % (pkg.name, pkg.tarball))
        return True
    return False


class Prefetcher:
    """Background tarball fetcher: downloads the sources of upcoming
    packages while the current one compiles. Dead/stub sources surface
    (and get seeded from the mirror) BEFORE their build starts instead of
    halting the run hours later. Touches only $SRC -- never the store
    (sqlite is not shared across threads); reports dead sources to a file
    the human can fix while the build goes on."""

    def __init__(self, srcdir, msg, report):
        import glob
        import queue
        import threading
        self.srcdir = srcdir
        self.msg = msg
        self.report = report
        self.queued = set()
        self.q = queue.Queue()
        self.dead = 0
        for part in glob.glob(os.path.join(srcdir, '*.part')):
            try:                # leftover of a download cut by a past exit
                os.remove(part)
            except OSError:
                pass
        threading.Thread(target=self._run, daemon=True).start()

    def push(self, pkg):
        if not pkg.tarball or pkg.name in self.queued:
            return
        self.queued.add(pkg.name)
        self.q.put(pkg)

    def _run(self):
        while True:
            pkg = self.q.get()
            try:
                self._fetch(pkg)
            except Exception:
                pass            # a prefetch problem must never kill the run

    def _fetch(self, pkg):
        path = os.path.join(self.srcdir, pkg.tarball)
        if os.path.exists(path) and tarball_ok(path):
            return
        # a stub on disk stays until a good download replaces it: see rescue
        # upstream first (canonical, keeps mirror load low), mirror second;
        # vcs pseudo-URLs (git|...) can only come from the mirror tarball
        if pkg.wget and not is_vcs_url(pkg.wget) \
                and fetch_url(pkg.wget, path):
            return
        if fetch_url(mirror_url(pkg.tarball), path):
            self.msg('prefetch %s: upstream dead, %s seeded from mirror\n'
                     % (pkg.name, pkg.tarball))
            return
        self.dead += 1
        self.msg('prefetch %s: SOURCE UNREACHABLE (%s)\n'
                 % (pkg.name, pkg.tarball))
        try:
            with open(self.report, 'a') as fh:
                fh.write('%s\t%s\t%s\t%s\n'
                         % (now(), pkg.name, pkg.tarball, pkg.wget))
        except OSError:
            pass


def fmt_dur(secs):
    secs = int(secs)
    h, rem = divmod(secs, 3600)
    m, s = divmod(rem, 60)
    if h:
        return '%dh%02dm' % (h, m)
    if m:
        return '%dm%02ds' % (m, s)
    return '%ds' % s


def ts_epoch(ts):
    """'YYYY-mm-dd HH:MM:SS' -> epoch seconds (0 if empty/unparseable)."""
    try:
        return time.mktime(time.strptime(ts, '%Y-%m-%d %H:%M:%S'))
    except (ValueError, OverflowError):
        return 0


class History:
    """Cross-run learning: per-package last build time, consecutive failure
    count and last status. Lives in its own file so --restart never wipes it."""

    def __init__(self, sdir, backend):
        self.use_sqlite = (HAVE_SQLITE if backend == 'auto'
                           else backend == 'sqlite')
        self.d = {}
        if self.use_sqlite:
            self.db = sqlite3.connect(os.path.join(sdir, 'history.db'))
            self.db.execute(
                'CREATE TABLE IF NOT EXISTS hist('
                'name TEXT PRIMARY KEY, last_seconds REAL DEFAULT 0, '
                'consec_fails INTEGER DEFAULT 0, last_status TEXT DEFAULT "", '
                'last_ts TEXT DEFAULT "")')
            self.db.commit()
        else:
            self.path = os.path.join(sdir, 'history.tsv')
            try:
                with open(self.path) as fh:
                    for line in fh:
                        f = line.rstrip('\n').split('\t')
                        if len(f) == 5:
                            self.d[f[0]] = {'last_seconds': float(f[1]),
                                            'consec_fails': int(f[2]),
                                            'last_status': f[3],
                                            'last_ts': f[4]}
            except OSError:
                pass

    def get(self, name):
        if self.use_sqlite:
            r = self.db.execute(
                'SELECT last_seconds,consec_fails,last_status,last_ts '
                'FROM hist WHERE name=?', (name,)).fetchone()
            if not r:
                return {'last_seconds': 0, 'consec_fails': 0,
                        'last_status': '', 'last_ts': ''}
            return {'last_seconds': r[0], 'consec_fails': r[1],
                    'last_status': r[2], 'last_ts': r[3]}
        return dict(self.d.get(name, {'last_seconds': 0, 'consec_fails': 0,
                                      'last_status': '', 'last_ts': ''}))

    def record(self, name, ok, seconds):
        h = self.get(name)
        secs = seconds if (ok and seconds > 0) else h['last_seconds']
        fails = 0 if ok else h['consec_fails'] + 1
        status = 'done' if ok else 'failed'
        if self.use_sqlite:
            self.db.execute(
                'INSERT OR REPLACE INTO hist'
                '(name,last_seconds,consec_fails,last_status,last_ts) '
                'VALUES(?,?,?,?,?)', (name, secs, fails, status, now()))
            self.db.commit()
        else:
            self.d[name] = {'last_seconds': secs, 'consec_fails': fails,
                            'last_status': status, 'last_ts': now()}
            tmp = self.path + '.tmp'
            with open(tmp, 'w') as fh:
                for n, r in sorted(self.d.items()):
                    fh.write('%s\t%g\t%d\t%s\t%s\n' % (
                        n, r['last_seconds'], r['consec_fails'],
                        r['last_status'], r.get('last_ts', '')))
            os.replace(tmp, self.path)

    def median_seconds(self, default=60.0):
        vals = []
        if self.use_sqlite:
            vals = [r[0] for r in self.db.execute(
                'SELECT last_seconds FROM hist WHERE last_seconds>0')]
        else:
            vals = [r['last_seconds'] for r in self.d.values()
                    if r['last_seconds'] > 0]
        if not vals:
            return default
        vals.sort()
        return vals[len(vals) // 2]

    def close(self):
        if self.use_sqlite:
            self.db.close()


def drive(store, order, pkgs, arch, wok, cache, sdir, seed, pkgsdir,
          simulate=False, fail=frozenset(), limit=0, no_stop=False,
          history=None, retries=2, retry_broken=False, broken_after=3,
          transient=frozenset(), skip=(), dlfail=frozenset(), max_consec=5,
          srcdir='/home/slitaz/src', prefetch_n=8):
    """Walk the order and cook every package once. A failure blocks nothing:
    the old package stays available from the seed, so one pass finds nearly
    every broken receipt instead of hiding its dependents behind a cascade.
    The state store is the only proof of a rebuild: $PKGS starts full."""
    universe = set(order)
    msg = sys.stderr.write

    # in-memory status mirror: drive is the single writer, and the hot paths
    # (ETA) would otherwise be thousands of store round-trips per package.
    status = store.statuses()

    def setst(name, st, **kw):
        status[name] = st
        store.set(name, st, **kw)

    # Receipt removed from the wok by a pull during the run
    for name in list(status):
        if name not in universe and status[name] != 'skipped':
            setst(name, 'skipped', culprit='gone-from-wok')

    def receipt_edited(name):
        """Receipt changed since this package was last cooked?"""
        if not history:
            return False
        try:
            rmt = os.stat(os.path.join(
                wok, pkgs[name].wokdir, 'receipt')).st_mtime
        except OSError:
            return False
        # whole seconds: last_ts has no fraction, a same-second cook isn't older
        return int(rmt) > ts_epoch(history.get(name)['last_ts'])

    def halt(reason):
        with open(os.path.join(sdir, 'stop'), 'w') as fh:
            fh.write('%s\t%s\n' % (now(), reason))
        msg('\n*** HALT: %s ***\n' % reason)

    logpath = os.path.join(sdir, 'run.log')
    med = history.median_seconds() if history else 60.0

    def expected(n):
        s = history.get(n)['last_seconds'] if history else 0
        return s if s > 0 else med

    def eta_remaining():
        return sum(expected(n) for n in order
                   if status.get(n) in ('todo', 'building'))

    if not simulate:
        pend = sum(1 for n in order if status.get(n) == 'todo')
        msg('  ETA (rough)  : %s for %d pending\n'
            % (fmt_dur(eta_remaining()), pend))

    recent = []
    last_snap = [0.0]

    def snap(current='', force=False):
        if not force and time.time() - last_snap[0] < 2:
            return
        last_snap[0] = time.time()
        store.flush()
        write_progress(sdir, store, arch, store.backend, current=current,
                       eta=int(eta_remaining()), recent=recent[-15:])

    def log_recent(name, st, secs=0):
        recent.append({'name': name, 'status': st, 'seconds': int(secs)})

    prefetcher = None
    if prefetch_n and not simulate:
        prefetcher = Prefetcher(srcdir, msg,
                                os.path.join(sdir, 'prefetch-dead.tsv'))

    def feed_prefetch(start):
        """Queue the tarballs of the next prefetch_n pending packages."""
        fed = 0
        for nx in order[start:]:
            if fed >= prefetch_n:
                break
            if status.get(nx) == 'todo' \
                    and not any(fnmatch.fnmatch(nx, p) for p in skip):
                prefetcher.push(pkgs[nx])
                fed += 1

    snap(force=True)
    built = 0
    consec = 0          # consecutive failures (any package)
    consec_dl = 0       # consecutive src-download failures (network down?)
    consec_fast = 0     # consecutive INSTANT failures (cook itself broken?)
    try:
        for pos, name in enumerate(order):
            st = status.get(name)
            if st == 'done':
                if not receipt_edited(name):
                    continue
                msg('recook %s (receipt edited)\n' % name)
            # intentional skip-list (globs): never build, never halt
            if any(fnmatch.fnmatch(name, pat) for pat in skip):
                setst(name, 'skipped', culprit='skip-list')
                log_recent(name, 'skipped')
                msg('skip   %s (skip-list)\n' % name)
                snap()
                continue
            # learned auto-skip: durably broken across past runs -- unless the
            # receipt was edited after the last failure (= someone fixed it)
            if history and not retry_broken:
                h = history.get(name)
                if h['last_status'] == 'failed' \
                        and h['consec_fails'] >= broken_after:
                    if not receipt_edited(name):
                        setst(name, 'skipped',
                              culprit='durably-broken x%d' % h['consec_fails'])
                        log_recent(name, 'skipped')
                        msg('skip   %s (durably broken x%d)\n'
                            % (name, h['consec_fails']))
                        snap()
                        continue
                    msg('retry  %s (durably broken x%d, but receipt edited)\n'
                        % (name, h['consec_fails']))

            # build, retrying on transient (network/fetch) failures
            base_attempts = store.get(name)['attempts']
            setst(name, 'building')
            if prefetcher:
                feed_prefetch(pos + 1)
            snap(current=name, force=True)
            ok = False
            dl_fail = False
            rescued = False
            tries = 0
            t0 = time.time()
            while True:
                tries += 1
                if simulate:
                    ok = (name not in fail and name not in dlfail
                          and not (name in transient and tries == 1))
                else:
                    rc = run_cook(name, logpath)
                    if rc == 127:   # no cook in PATH: environment, not package
                        setst(name, 'failed', culprit='no-cook')
                        halt('cook not found in PATH')
                        return
                    ok = (rc == 0) and (name not in broken_set(cache))
                if ok:
                    break
                # transient (network blink)? retry BEFORE classifying as a
                # dead source -- the patterns overlap (DNS down looks dead)
                tr = (name in transient) if simulate else is_transient(logpath)
                if tr and tries < retries:
                    msg('retry  %s (transient, %d/%d)\n'
                        % (name, tries, retries))
                    continue
                # missing/stub tarball? seed it from the sources mirror and
                # re-cook once instead of failing on a dead upstream URL
                if not simulate and not rescued \
                        and rescue_source(pkgs[name], srcdir, msg):
                    rescued = True
                    continue
                dl_fail = (name in dlfail) if simulate else is_dl_fail(logpath)
                break
            secs = 0.0 if simulate else time.time() - t0
            # a split recook that dies halfway can leave a package missing:
            # its consumers still need the old one to build
            if not ok and not simulate:
                for out in seed.restore(pkgs[name].outputs(), pkgsdir):
                    msg('seed   %s restored after %s failed\n' % (out, name))
            setst(name, 'done' if ok else 'failed', seconds=secs,
                  attempts=base_attempts + tries,
                  culprit='src-download' if dl_fail else '')
            if history:
                history.record(name, ok, secs)
            log_recent(name, 'dlfail' if dl_fail
                       else ('done' if ok else 'failed'), secs)
            tag = 'ok' if ok else ('DLFAIL' if dl_fail else 'FAIL')
            msg('%-6s %s%s\n' % (tag, name,
                                 '' if simulate else ' (%s)' % fmt_dur(secs)))
            snap()
            built += 1
            consec = 0 if ok else consec + 1
            consec_dl = (consec_dl + 1 if dl_fail else 0) if not ok else 0
            # a real build failure takes time; dying instantly, many times
            # in a row, means cook/the environment is broken in a way the
            # toolchain canary can't see (seen: a cook deployed with its
            # @@PREFIX@@ placeholders unsubstituted -- 100+ 0s failures
            # while the canary kept saying the toolchain was healthy)
            fast = (not ok) and not simulate and secs < 5
            consec_fast = consec_fast + 1 if fast else 0

            # only a broken ENVIRONMENT stops the run
            if not ok and not no_stop and max_consec:
                if consec_dl >= max_consec:
                    halt('%d source-download failures in a row ("%s" last)'
                         ' -- network down?' % (consec_dl, name))
                    return
                # 3x the generic threshold: small leaf packages can fail
                # fast for real, individual reasons (a 2s gcc error is
                # legitimate) -- systemic cook breakage hits dozens in a row
                if consec_fast >= 3 * max_consec:
                    halt('%d instant (<5s) failures in a row ("%s" last) -- '
                         'cook itself broken? check %s'
                         % (consec_fast, name,
                            os.path.join(conf('LOGS') or '/home/slitaz/log',
                                         name + '.log')))
                    return
                if consec >= max_consec:
                    # a streak is only an environment problem if the toolchain
                    # really is broken -- 5 siblings dying of the same receipt
                    # bug is just an unlucky cluster
                    probs = [] if simulate else toolchain_canary()
                    if probs:
                        halt('%d failures in a row ("%s" last): %s'
                             % (consec, name, '; '.join(probs)))
                        return
                    msg('note   %d failures in a row but the toolchain is '
                        'healthy: going on\n' % consec)
                    consec = 0

            if limit and built >= limit:
                msg('reached --limit %d\n' % limit)
                break
    finally:
        # always leave a fresh snapshot + flushed store, even on ^C/halt
        snap(force=True)


def write_progress(sdir, store, arch, backend, current='', eta=-1,
                   recent=None):
    """Snapshot the run state to progress.json for the web dashboard.

    A bare call (no current/eta/recent, e.g. from --status) must not blank the
    live panel: reuse the previous snapshot's current/eta/recent in that case."""
    c = store.counts()
    halt = None
    try:
        with open(os.path.join(sdir, 'stop')) as fh:
            halt = fh.read().strip()
    except OSError:
        pass
    if recent is None:
        try:
            with open(os.path.join(sdir, 'progress.json')) as fh:
                prev = json.load(fh)
            recent = prev.get('recent', [])
            if not current:
                current = prev.get('current', '')
            if eta < 0:
                eta = prev.get('eta_seconds', -1)
        except (OSError, ValueError):
            recent = []
    data = {'arch': arch, 'backend': backend, 'updated': now(),
            'total': sum(c.values()), 'counts': c, 'current': current,
            'eta_seconds': eta, 'halt': halt, 'recent': recent or []}
    tmp = os.path.join(sdir, 'progress.json.tmp')
    try:
        with open(tmp, 'w') as fh:
            json.dump(data, fh)
        os.replace(tmp, os.path.join(sdir, 'progress.json'))
    except OSError:
        pass


def cmd_status(store, sdir=None):
    c = store.counts()
    total = sum(c.values())
    out = sys.stdout.write
    out('cook-full state (%s)\n' % store.backend)
    if sdir:
        try:
            with open(os.path.join(sdir, 'stop')) as fh:
                out('  HALTED   : %s\n' % fh.read().strip())
        except OSError:
            pass
    out('  total    : %d\n' % total)
    for s in STATUSES:
        if c[s]:
            out('  %-8s : %d\n' % (s, c[s]))
    # split failures into dead-source (src-download) vs real build failures
    failed = store.by_status('failed')
    dl = [n for n in failed if store.get(n)['culprit'] == 'src-download']
    build = [n for n in failed if n not in dl]
    if dl:
        out('  src-download (dead URL): %s%s\n'
            % (' '.join(dl[:20]), ' ...' if len(dl) > 20 else ''))
    if build:
        out('  failed: %s%s\n'
            % (' '.join(build[:20]), ' ...' if len(build) > 20 else ''))
    names = store.by_status('skipped', 20)
    if names:
        out('  skipped: %s%s\n' % (' '.join(names),
                                   ' ...' if c['skipped'] > 20 else ''))
    if sdir:
        try:
            with open(os.path.join(sdir, 'prefetch-dead.tsv')) as fh:
                dead = [l.split('\t')[1] for l in fh if '\t' in l]
            if dead:
                out('  prefetch found unreachable sources: %s\n'
                    % ' '.join(sorted(set(dead))))
        except OSError:
            pass


# Self-contained dashboard palette, shared look with web/cook-full.cgi (kept
# duplicated: the report ships as one standalone .html, no external assets).
REPORT_CSS = '''
:root{--bg:#1c1c1c;--fg:#dcdcdc;--dim:#8a8a8a;--accent:#ff8700;--ok:#5fd75f;
--err:#ff5f5f;--warn:#ffaf00;--card:#262626;--bar:#333}
*{box-sizing:border-box}
body{margin:0;background:var(--bg);color:var(--fg);
font:14px/1.5 "DejaVu Sans Mono",monospace}
header{background:#111;padding:14px 20px;border-bottom:2px solid var(--accent)}
header h1{margin:0;font-size:20px;color:var(--accent)}
header span{color:var(--dim);font-size:13px}
main{padding:20px;max-width:980px;margin:0 auto}
h2{font-size:13px;color:var(--dim);text-transform:uppercase;letter-spacing:1px;
margin:26px 0 8px;border-bottom:1px solid #333;padding-bottom:4px}
.barwrap{background:var(--bar);border-radius:6px;height:26px;overflow:hidden;
margin:6px 0 4px}
.bar{height:100%;background:linear-gradient(90deg,#4a7,#5fd75f);text-align:right;
color:#06250a;font-weight:bold;padding-right:8px;line-height:26px;white-space:nowrap}
.chips{display:flex;flex-wrap:wrap;gap:10px;margin:14px 0}
.chip{background:var(--card);border-radius:6px;padding:8px 14px;min-width:92px}
.chip b{display:block;font-size:20px}
.chip.done b{color:var(--ok)}.chip.failed b{color:var(--err)}
.chip.skipped b{color:var(--warn)}
.chip.todo b{color:var(--dim)}.chip.building b{color:var(--accent)}
table{border-collapse:collapse;width:100%;margin:4px 0}
td,th{text-align:left;padding:4px 10px;border-bottom:1px solid #2c2c2c;
vertical-align:top}
th{color:var(--dim);font-weight:normal;font-size:12px}
td.pkg{color:var(--fg);white-space:nowrap}
td.note{color:#bdbdbd}
.cat{display:flex;flex-wrap:wrap;gap:8px}
.cat span{background:var(--card);border-radius:5px;padding:4px 10px;font-size:13px}
.cat b{color:var(--ok)}
.badge{display:inline-block;border-radius:4px;padding:1px 8px;font-size:12px;
font-weight:bold;color:#111;white-space:nowrap}
.a-fixed{background:var(--ok)}.a-fix{background:var(--warn)}
.a-wait{background:#5fafff}.a-drop{background:var(--err)}
.a-auto{background:#555;color:#ddd}
footer{color:var(--dim);padding:18px 20px;text-align:center;font-size:12px}
'''


def _auto_cat(name, logdir):
    """Best-effort failure category from the package's cook log tail."""
    lp = os.path.join(logdir, name + '.log')
    if not os.path.exists(lp):
        return ''
    if is_dl_fail(lp):
        return 'source download (dead URL)'
    if is_transient(lp):
        return 'transient / network'
    return 'build error'


def _cat_index(pkgsinfo):
    """{pkgname: CATEGORY} from packages.info (col 3); {} if absent."""
    idx = {}
    if pkgsinfo and os.path.exists(pkgsinfo):
        with open(pkgsinfo) as fh:
            for l in fh:
                f = l.rstrip('\n').split('\t')
                if len(f) >= 3:
                    idx[f[0]] = f[2]
    return idx


def build_report(store, triage, arch, pkgsinfo, logdir, deps_count):
    """Render the standalone shareable HTML status report (string)."""
    esc = html.escape
    c = store.counts()
    total = sum(c.values()) or 1
    done = c.get('done', 0)
    pct = 100.0 * done / total
    b = []

    b.append('<div class="barwrap"><div class="bar" style="width:%.1f%%">%.1f%%'
             '</div></div>' % (max(pct, 6), pct))
    order = ('done', 'failed', 'skipped', 'building', 'todo')
    b.append('<div class="chips">%s</div>' % ''.join(
        '<div class="chip %s"><b>%d</b>%s</div>' % (k, c.get(k, 0), k)
        for k in order))

    # culprit encodes WHY a package was skipped
    skipped = store.by_status('skipped')
    buckets = {'skip': 0, 'broken': 0, 'gone': 0}
    for n in skipped:
        cul = store.get(n)['culprit'] or ''
        if cul.startswith('durably-broken'):
            buckets['broken'] += 1
        elif cul == 'gone-from-wok':
            buckets['gone'] += 1
        else:
            buckets['skip'] += 1

    # --- For developers: a concrete action plan, ranked by impact ---
    fixes = [(g, note) for g, a, _c, note in triage if a == 'fix']
    waits = [(g, note) for g, a, _c, note in triage if a == 'wait']
    # A failure blocks nothing during the run (the seed still serves the old
    # package), so rank by how many packages build against it.
    levers = sorted(((n, deps_count.get(n, 0))
                     for n in store.by_status('failed')),
                    key=lambda kv: (-kv[1], kv[0]))
    levers = [x for x in levers if x[1]][:12]
    b.append('<h2>For developers &mdash; what to do</h2>')
    todo = []
    if fixes:
        todo.append('<p class="note"><b>1. Known fixes to apply</b> '
                    '(the what-to-do is known):</p><table><tr><th>package</th>'
                    '<th>action</th></tr>%s</table>' % ''.join(
            '<tr><td class="pkg">%s</td><td class="note">%s</td></tr>'
            % (esc(g), esc(note)) for g, note in fixes))
    if levers:
        todo.append('<p class="note"><b>2. Fix first</b> (failed packages '
                    'that others build against):</p>'
                    '<table><tr><th>package</th><th>used by</th></tr>%s'
                    '</table>' % ''.join(
            '<tr><td class="pkg">%s</td><td><b>%d</b> packages</td></tr>'
            % (esc(r), n) for r, n in levers))
    if waits:
        todo.append('<p class="note"><b>3. Decisions to make</b> (fix or drop '
                    '&mdash; not a code bug):</p><table><tr>'
                    '<th>package</th><th>context</th></tr>%s</table>' % ''.join(
            '<tr><td class="pkg">%s</td><td class="note">%s</td></tr>'
            % (esc(g), esc(note)) for g, note in waits))
    if buckets['broken']:
        todo.append('<p class="note">+ <b>%d packages failing repeatedly</b> to '
                    'investigate one by one (log: '
                    '<code>%s/&lt;package&gt;.log</code>).</p>'
                    % (buckets['broken'], esc(logdir)))
    b.append(''.join(todo) or '<p style="color:var(--dim)">&mdash;</p>')

    # Done -- summary by receipt CATEGORY (from packages.info), no name dump
    cats = {}
    catidx = _cat_index(pkgsinfo)
    for n in store.by_status('done'):
        cats[catidx.get(n, '(other)')] = cats.get(catidx.get(n, '(other)'), 0) + 1
    b.append('<h2>Done &mdash; %d packages</h2>' % done)
    if cats:
        b.append('<div class="cat">%s</div>' % ''.join(
            '<span>%s <b>%d</b></span>' % (esc(k), v)
            for k, v in sorted(cats.items(), key=lambda kv: -kv[1])))

    def rows(names, fallback_logcat=False):
        r = []
        for n in names:
            t = triage_for(n, triage)
            if t:
                action, cat, note = t
            elif fallback_logcat:
                action, cat, note = 'auto', _auto_cat(n, logdir), ''
            else:
                action, cat, note = 'auto', '', ''
            r.append('<tr><td class="pkg">%s</td><td><span class="badge a-%s">'
                     '%s</span></td><td>%s</td><td class="note">%s</td></tr>'
                     % (esc(n), esc(action), esc(action), esc(cat), esc(note)))
        return ('<table><tr><th>package</th><th>action</th><th>category</th>'
                '<th>note</th></tr>%s</table>' % ''.join(r)) if r else \
               '<p style="color:var(--dim)">&mdash;</p>'

    # Failures -- with fix idea / wait reason (triage), log-category fallback
    failed = store.by_status('failed')
    b.append('<h2>Failures &amp; possible fixes &mdash; %d</h2>' % len(failed))
    b.append(rows(failed, fallback_logcat=True))

    # Drops -- curated only (re-homed pkgs are no longer in the db)
    drops = [(g, cat, note) for g, a, cat, note in triage if a == 'drop']
    b.append('<h2>Dropped &amp; reasons &mdash; %d</h2>' % len(drops))
    if drops:
        b.append('<table><tr><th>package</th><th>category</th><th>reason</th>'
                 '</tr>%s</table>' % ''.join(
            '<tr><td class="pkg">%s</td><td>%s</td><td class="note">%s</td></tr>'
            % (esc(g), esc(cat), esc(note)) for g, cat, note in drops))
    else:
        b.append('<p style="color:var(--dim)">&mdash;</p>')

    # Skipped detail: one reason per family, no name dump beyond the skip-list
    b.append('<h2>Skipped &mdash; %d</h2>' % len(skipped))
    b.append('<table><tr><th>reason</th><th>n</th><th>explanation</th></tr>%s'
             '</table>' % ''.join(
        '<tr><td class="pkg">%s</td><td>%d</td><td class="note">%s</td></tr>'
        % (esc(label), buckets[key], esc(expl))
        for key, label, expl in (
            ('skip', 'intentionally excluded', 'data/skip or --skip'),
            ('broken', 'given up (fails repeatedly)',
             'fail on every attempt -- candidates to fix or drop'),
            ('gone', 'gone from the wok', 'receipt removed during the run'))
        if buckets[key]))

    b.append('<footer>arch %s &middot; %d packages &middot; generated %s '
             '&middot; SliTaz cook-full</footer>'
             % (esc(arch), total, esc(now())))

    return ('<!DOCTYPE html><html lang="en"><head><meta charset="utf-8">'
            '<meta name="viewport" content="width=device-width,initial-scale=1">'
            '<title>Cook-Full — report</title><style>%s</style></head><body>'
            '<header><h1>Cook-Full — report</h1>'
            '<span>full SliTaz build status (%s)</span></header>'
            '<main>%s</main></body></html>'
            % (REPORT_CSS, esc(arch), ''.join(b)))


def cmd_report(store, out_path, triage, arch, pkgsinfo, logdir, deps_count):
    page = build_report(store, triage, arch, pkgsinfo, logdir, deps_count)
    with open(out_path, 'w') as fh:
        fh.write(page)
    sys.stderr.write('cook-full: report written to %s\n' % out_path)


def main():
    ap = argparse.ArgumentParser(
        prog='cook-full',
        description='Recook the whole wok in a dedicated build environment.')
    ap.add_argument('--arch', help='target arch (default: cook.conf/host)')
    ap.add_argument('--wok', help='wok path (default: cook.conf $WOK)')
    ap.add_argument('--root', default='/',
                    help='root holding the base install (default: /)')
    ap.add_argument('--seed-dir', default=DEFAULT_SEED,
                    help='read-only snapshot of the prod packages '
                         '(default: %s)' % DEFAULT_SEED)
    ap.add_argument('--state-dir',
                    help='state directory (default: $CACHE/cook-full)')
    ap.add_argument('--dry-run', action='store_true',
                    help='print the computed build order, cook nothing')
    ap.add_argument('--cycles', action='store_true',
                    help='with --dry-run: list where dependency cycles were cut')
    ap.add_argument('--check', action='store_true',
                    help='with --dry-run: verify the order invariant')
    ap.add_argument('--status', action='store_true',
                    help='print the saved build state and exit')
    ap.add_argument('--report', nargs='?', const='cook-full-report.html',
                    default=None, metavar='PATH',
                    help='write a standalone shareable HTML status report to '
                         'PATH (default cook-full-report.html) and exit; reads '
                         'the live db, no run needed')
    ap.add_argument('--restart', action='store_true',
                    help='wipe saved state before running')
    ap.add_argument('--backend', choices=('auto', 'sqlite', 'tsv'),
                    default='auto', help='state backend (default: auto)')
    ap.add_argument('--simulate', action='store_true',
                    help='drive the loop without cooking (test state machine); '
                         'uses its own state dir, never touches a real run')
    ap.add_argument('--fail', default='',
                    help='with --simulate: comma list of pkgs to mark failed')
    ap.add_argument('--limit', type=int, default=0,
                    help='stop after cooking N packages (testing)')
    ap.add_argument('--skip', default='',
                    help='comma list of globs to skip, e.g. "linux*" '
                         '(merged with data/skip)')
    ap.add_argument('--no-stop', action='store_true',
                    help='never halt on failure streaks')
    ap.add_argument('--consec', type=int, default=5,
                    help='check the environment after N consecutive failures '
                         '(def 5, 0=off)')
    ap.add_argument('--no-canary', action='store_true',
                    help='skip the pre-run toolchain sanity check')
    ap.add_argument('--prefetch', type=int, default=8,
                    help='download the next N source tarballs in the '
                         'background while building (def 8, 0=off)')
    ap.add_argument('--continue', dest='cont', action='store_true',
                    help='clear a previous stop sentinel and resume')
    ap.add_argument('--retries', type=int, default=2,
                    help='max attempts on transient (network) failures (def 2)')
    ap.add_argument('--retry-broken', action='store_true',
                    help='also retry packages marked durably broken')
    ap.add_argument('--transient', default='',
                    help='with --simulate: pkgs that fail once then succeed')
    ap.add_argument('--dlfail', default='',
                    help='with --simulate: pkgs that fail as source-download')
    args = ap.parse_args()

    arch = args.arch or conf('ARCH') or host_arch()
    wok = args.wok or conf('WOK') or '/home/slitaz/wok'
    cache = conf('CACHE') or '/home/slitaz/cache'
    pkgsdir = conf('PKGS') or '/home/slitaz/packages'
    # --simulate gets its own state dir: it marks packages done/failed, so
    # sharing the real run's state.db would corrupt a build in progress.
    sdir = args.state_dir or os.path.join(
        cache, 'cook-full-sim' if args.simulate else 'cook-full')
    here = os.path.dirname(os.path.realpath(__file__))

    if args.status:
        os.makedirs(sdir, exist_ok=True)
        store = open_store(sdir, args.backend)
        cmd_status(store, sdir)
        write_progress(sdir, store, arch, store.backend)
        store.close()
        return

    if not os.path.isdir(wok):
        sys.exit('cook-full: wok not found: %s' % wok)

    # The startup work (sourcing the receipts to build the dep graph, then
    # ordering) takes a while with no output otherwise -- announce each phase.
    # The mtime-keyed scan cache makes the usual fix-and-continue restart ~1s.
    msg = sys.stderr.write
    t0 = time.time()
    cachefile = None
    try:
        os.makedirs(sdir, exist_ok=True)
        cachefile = os.path.join(sdir, 'scan-cache.tsv')
    except OSError:
        pass        # e.g. --dry-run outside the chroot: scan without cache
    msg('cook-full: scanning wok %s (arch=%s)...\n' % (wok, arch))
    pkgs = scan_wok(wok, arch, cachefile)
    msg('  %d receipts scanned in %s\n' % (len(pkgs), fmt_dur(time.time() - t0)))

    seed = Seed(args.seed_dir)
    installed = read_installed(args.root)
    t1 = time.time()
    msg('  computing build order...\n')
    need, unknown, orphans = build_need(pkgs, arch, installed, seed)
    order, breaks = compute_order(need)
    msg('  ordered %d, cycles cut %d in %s\n'
        % (len(order), len(breaks), fmt_dur(time.time() - t1)))

    if args.report is not None:
        store = open_store(sdir, args.backend)
        triage = load_triage(os.path.join(here, 'data', 'triage.tsv'))
        pkgsinfo = os.path.join(pkgsdir, 'packages.info')
        logdir = conf('LOGS') or '/home/slitaz/log'
        cmd_report(store, args.report, triage, arch, pkgsinfo, logdir,
                   dependents(need))
        store.close()
        return

    if args.dry_run:
        for name in order:
            print(name)
        msg('\n')
        msg('cook-full --dry-run  arch=%s  wok=%s\n' % (arch, wok))
        msg('  receipts scanned : %d\n' % len(pkgs))
        msg('  arch-included    : %d\n' % len(need))
        msg('  split outputs    : %d\n'
            % sum(len(pkgs[n].split) for n in need))
        msg('  base installed   : %d (%s)\n' % (len(installed), args.root))
        msg('  seed             : %s\n'
            % ('%d packages (%s)' % (len(seed.rows), args.seed_dir)
               if seed.present else 'NONE (%s), DEPENDS ignored in the order'
               % args.seed_dir))
        msg('  cycles cut       : %d\n' % len(breaks))
        msg('  unknown bdeps    : %d\n' % len(unknown))
        for dep in sorted(unknown):
            msg('    %s <- %s\n' % (dep, ' '.join(sorted(unknown[dep]))))
        msg('  no receipt       : %d seed packages have no receipt here\n'
            % len(orphans))
        if args.check:
            pos = {n: i for i, n in enumerate(order)}
            cut = {n for n, _ in breaks}
            late = [(n, d) for n in order for d in need[n]
                    if pos[d] > pos[n] and n not in cut]
            msg('  order check      : %s (%d violations)\n'
                % ('OK' if not late else 'FAILED', len(late)))
            for n, d in late[:20]:
                msg('    %s before %s\n' % (n, d))
        if breaks and args.cycles:
            msg('\n  cycles cut at (package <- deps not rebuilt yet):\n')
            for name, unmet in breaks:
                msg('    %s <- %s\n' % (name, ' '.join(unmet)))
        return

    # A real run replaces every package of $PKGS: the guards below keep it out
    # of the production cooker, which has no seed, and away from any online
    # mirror, which would make cook pull packages from outside the run.
    if not args.simulate:
        if not seed.present:
            sys.exit('cook-full: no seed in %s: not a recook environment, '
                     'refusing to run' % args.seed_dir)
        if not mirror_is_local(args.root):
            sys.exit('cook-full: %svar/lib/tazpkg/mirror must be a local path'
                     % args.root)

    stopfile = os.path.join(sdir, 'stop')
    if os.path.exists(stopfile) and not (args.cont or args.restart):
        with open(stopfile) as fh:
            reason = fh.read().strip()
        sys.exit('cook-full: halted last run -> %s\n'
                 '  address it, then re-run with --continue (or '
                 '--restart)' % reason)

    skip_path = os.path.join(here, 'data', 'skip')
    skip = load_skip(skip_path if os.path.exists(skip_path) else None, args.skip)

    # pre-flight canary: refuse to start cooking with a broken toolchain
    # (every package would fail one by one until a halt threshold trips)
    if not args.simulate and not args.no_canary:
        probs = toolchain_canary()
        if probs:
            sys.exit('cook-full: build environment is broken, not starting:\n'
                     + ''.join('  - %s\n' % p for p in probs)
                     + '  fix it (or --no-canary to override)')

    # one cook-full per state dir: a second concurrent run (forgotten tmux,
    # double --continue) would silently corrupt the state
    lockfile = os.path.join(sdir, 'pid')
    if os.path.exists(lockfile):
        try:
            with open(lockfile) as fh:
                pid = int(fh.read().strip())
            os.kill(pid, 0)
            sys.exit('cook-full: already running (pid %d, %s)'
                     % (pid, lockfile))
        except (ValueError, OSError):
            pass        # stale lock: no such process
    with open(lockfile, 'w') as fh:
        fh.write('%d\n' % os.getpid())

    # wiped under the lock, so a --restart can't pull the state from a live run
    if args.restart:
        for f in ('state.db', 'state.tsv', 'stop', 'progress.json'):
            try:
                os.remove(os.path.join(sdir, f))
            except OSError:
                pass
    elif args.cont and os.path.exists(stopfile):
        os.remove(stopfile)

    store = open_store(sdir, args.backend)
    store.sync_order(order)
    history = History(sdir, args.backend)
    fail = set(x for x in args.fail.split(',') if x)
    transient = set(x for x in args.transient.split(',') if x)
    dlfail = set(x for x in args.dlfail.split(',') if x)
    msg('cook-full %s  arch=%s  backend=%s  order=%d  cycles cut=%d\n'
        % ('SIMULATE' if args.simulate else 'run', arch, store.backend,
           len(order), len(breaks)))
    msg('  streak halts: %s, consec>=%s  | retries: %d\n'
        % ('OFF (--no-stop)' if args.no_stop else 'ON',
           args.consec or 'off', args.retries))
    if skip:
        msg('  skip-list   : %s\n' % ' '.join(sorted(skip)))
    try:
        drive(store, order, pkgs, arch, wok, cache, sdir, seed, pkgsdir,
              simulate=args.simulate, fail=fail, limit=args.limit,
              no_stop=args.no_stop, history=history, retries=args.retries,
              retry_broken=args.retry_broken, transient=transient, skip=skip,
              dlfail=dlfail, max_consec=args.consec,
              srcdir=conf('SRC') or '/home/slitaz/src',
              prefetch_n=args.prefetch)
        msg('\n')
        cmd_status(store, sdir)
    finally:
        history.close()
        store.close()
        try:
            os.remove(lockfile)
        except OSError:
            pass


if __name__ == '__main__':
    main()
