#!/usr/bin/python3
# Generate debian/copyright for gnulib from the license notices in the
# file headers of the source tree.
#
# Copyright 2026 Boyuan Yang <byang@debian.org>
# License: GPL-3+
#
# Usage: run from the top of the source tree:
#   debian/copyright-gen/gen-copyright [--check | --list]
# --check fails if debian/copyright is not up to date; --list prints the
# license, its source and the copyright holders of every file instead.
#
# How the license of a file is determined, most specific first:
#   1. paragraphs in debian/copyright-gen/manual.dep5 (hand-written);
#   2. the license notice in the file header (or, failing that, at the
#      end of the file, as in ChangeLog);
#   3. the License field of the module description(s) in modules/ that
#      list the file (the weakest license wins, as documented in the
#      gnulib manual, section "Copyright"; test modules default to GPL);
#   4. the directory defaults documented in the same manual section
#      (doc/: GFDL, m4/: unlimited, modules/: FSF all-permissive);
#   5. the GPL-3+ default of "Files: *".
# Files whose header has a Copyright line but no recognised license
# notice are an error: classify them in manual.dep5.

import collections
import fnmatch
import os
import re
import sys

GENDIR = 'debian/copyright-gen'
FSF = 'Free Software Foundation, Inc.'
DEFAULT = 'GPL-3+'
HEAD_BYTES = 8000

# --- license notice recognition ------------------------------------------

COMMENT = re.compile(r'^\s*(?:/\*+|\*+/|\*+|#+|dnl\b|@c\b|@comment\b|;+|%+|--|//|"|rem\b|!)?', re.M)


def flatten(text):
    t = COMMENT.sub(' ', text)
    t = re.sub(r'\*/|/\*|\\\n|["\\]', ' ', t)  # comment ends, quoted shell/make lines
    return re.sub(r'\s+', ' ', t)


LATER = r'(?:,|;)? *or *(?:\(at your option\) *)?(?:any )?later version'
GPLNAME = re.compile(r'GNU (Lesser |Library )?General Public License')


def gpl_family(t):
    """Return (position, license) of the first GPL-family notice in t."""
    m = re.search(r'LGPLv3\+ or the GNU GPLv2\+|under (?:the )?terms of either:?', t)
    if m and re.search(r'Lesser General Public License.{0,120}version 3', t[m.start():m.start() + 600]):
        return m.start(), 'LGPL-3+ or GPL-2+'
    for m in GPLNAME.finditer(t):
        window = t[m.start():m.start() + 250]
        v = re.search(r'(?:version |v)(\d(?:\.\d)?)(?: of the License)?(' + LATER + r'| or later)?', window)
        if not v:
            continue
        name = 'LGPL' if m.group(1) else 'GPL'
        return m.start(), f'{name}-{v.group(1)}{"+" if v.group(2) else ""}'
    return None


EXCEPTIONS = [
    ('Autoconf', r'configur\w* script generated by Autoconf'),
    ('Texinfo', r'when this file is read by TeX when processing a Texinfo source'),
    ('Bison', r'Bison parser skeleton'),
    ('Libtool', r'GNU Libtool'),
]

SIMPLE = [
    ('Unicode-3.0', r'UNICODE LICENSE V3'),
    ('license-text', r'Everyone is permitted to copy and distribute verbatim copies of this '
                     r'license document,? but changing it is not allowed'),
    ('FSFULLR', r'unlimited permission to copy and/or distribute it, with or without '
                r'modifications, as long as this notice is preserved'),
    ('FSFUL', r'unlimited permission to (?:use, )?copy, distribute,? and modify it'),
    ('FSFAP', r'Copying and distribution of this file, with or without modification, '
              r'(?:are permitted in any medium without royalty|in any medium, are permitted '
              r'without royalty|are permitted) provided'),
    ('public-domain', r'(?:This|The) (?:file|code|implementation|program)[^.]{0,80}? '
                      r'(?:is|has been) (?:hereby )?(?:placed |released )?(?:in|into) the public domain'
                      r'|(?:placed|released) (?:in|into) the public domain'),
]


# Third-party permissive notices kept below the FSF notice, possibly in
# the middle of the file (e.g. fdlibm code in lib/trigl.c).
EXTRAS = [
    ('BSD-3-clause', r'Redistribution and use in source and binary forms, with or without '
                     r'modification, are permitted provided that the following conditions are met'),
    ('ISC', r'Permission to use, copy, modify, and distribute this software for any purpose '
            r'with or without fee is hereby granted, provided that the above copyright notice '
            r'and this permission notice appear in all copies'),
    ('SunPro', r'Permission to use, copy, modify, and distribute this software is freely '
               r'granted, provided that this notice is preserved'),
    ('public-domain', r'This code is hereby placed in the public domain'),
]


def extras(text):
    """Return [(license, holders)] of third-party notices anywhere in text."""
    out = []
    if not re.search(r'Redistribution and use|Permission to use|hereby placed in the public', text):
        return out
    for name, rx in EXTRAS:
        lines = text.split('\n')
        for i in range(len(lines)):
            window = '\n'.join(lines[max(0, i - 12):i + 12])
            if i % 12 or not re.search(rx, flatten(window)):
                continue
            if name == 'BSD-3-clause':
                w = flatten('\n'.join(lines[i:i + 40]))
                if re.search(r'advertising materials', w):
                    name = 'BSD-4-clause'
                elif not re.search(r'Neither the name', w):
                    name = 'BSD-2-clause'
            out.append((name, holders_of(window)))
            break
    return out


def classify(text):
    """Return the DEP-5 license name of the first license notice in a
    header, or None.  Unknown special exceptions and GFDL notices with
    invariant sections or cover texts also give None."""
    t = flatten(text)
    found = []
    for name, rx in SIMPLE:
        m = re.search(rx, t, re.I if name == 'public-domain' else 0)
        if m:
            found.append((m.start(), name))
    m = re.search(r'GNU Free Documentation License,? Version 1\.3 or any later', t)
    if m:
        niv = re.search(r'no Invariant Sections,? (?:with )?no Front-Cover Texts,? and '
                        r'(?:with )?no Back-Cover Texts', t[m.start():m.start() + 400])
        found.append((m.start(), 'GFDL-NIV-1.3+' if niv else None))
    g = gpl_family(t)
    if g:
        pos, lic = g
        if re.search(r'special exception', t[pos:pos + 1500], re.I):
            name = next((n for n, rx in EXCEPTIONS if re.search(rx, t[pos:pos + 1500])), None)
            lic = f'{lic} with {name} exception' if name else None
        found.append((pos, lic))
    return min(found)[1] if found else None


# --- copyright holders -----------------------------------------------------

COPYLINE = re.compile(r'Copyright\s*(?:\(C\)|\(c\)|©|@copyright\{\})?\s*'
                      r'((?:(?:19|20)\d\d(?:\s*(?:-|–|--)\s*(?:19|20)?\d\d)?\s*,?\s*)+)(.*)')


def years_of(s):
    ys = set()
    for a, b in re.findall(r'((?:19|20)\d\d)(?:\s*(?:-|–|--)\s*((?:19|20)?\d\d))?', s):
        a = int(a)
        if b:
            b = int(b) if len(b) == 4 else a // 100 * 100 + int(b)
            ys.update(range(a, b + 1))
        else:
            ys.add(a)
    return ys


def holders_of(text):
    """Return {holder: set(years)} from the Copyright lines of a header."""
    lines = [COMMENT.sub('', l).strip() for l in text.split('\n')]
    out = collections.defaultdict(set)
    for i, l in enumerate(lines):
        m = COPYLINE.search(l)
        if not m:
            continue
        # join continuation lines: more years, or the holder on the next line
        j = i
        while j + 1 < len(lines) and lines[j + 1]:
            who = m.group(2).strip()
            if who and not who.endswith(',') and who not in ('Free', 'Free Software') \
                    and not re.fullmatch(r'[\d,\s–-]+', who):
                break
            j += 1
            m = COPYLINE.search(m.group(0) + ' ' + lines[j])
        who = re.sub(r'\s+', ' ', m.group(2).strip())
        if '$' in who or '%' in who:
            continue  # a copyright line printed by a script or format string
        if re.match(r'^Free\b', who) or 'Free Software Foundation' in who:
            who = FSF
        else:
            who = re.sub(r'^by\s+', '', who)
            who = re.sub(r'\s*All rights reserved\.?$', '', who).rstrip(' ,')
            if who.endswith('.') and not re.search(r'\b(?:Inc|Ltd|Co|Corp)\.$', who):
                who = who[:-1]
        if who and not re.search(r'copyright-holder|YEAR|<.*holder', who):
            out[who] |= years_of(m.group(1))
    return out


def fmt_years(ys):
    ys = sorted(ys)
    parts, i = [], 0
    while i < len(ys):
        j = i
        while j + 1 < len(ys) and ys[j + 1] == ys[j] + 1:
            j += 1
        parts.append(str(ys[i]) if i == j else f'{ys[i]}-{ys[j]}')
        i = j + 1
    return ', '.join(parts)


def fmt_copyright(holders):
    if not holders:
        holders = {FSF: set()}
    keys = sorted(holders, key=lambda h: (h != FSF, h))
    out = []
    for h in keys:
        y = fmt_years(holders[h])
        out.append(f'{y} {h}' if y else h)
    return out


# --- modules ---------------------------------------------------------------

MODULE_LICENSE = {
    'GPL': 'GPL-3+', 'GPLv3+': 'GPL-3+', 'GPLed build tool': 'GPL-3+',
    'GPLv2+': 'GPL-2+', 'GPLv2+ build tool': 'GPL-2+',
    'LGPL': 'LGPL-3+', 'LGPLv3+': 'LGPL-3+', 'LGPLv2+': 'LGPL-2.1+',
    'LGPLv3+ or GPLv2+': 'LGPL-3+ or GPL-2+',
    'unlimited': 'FSFULLR', 'public domain': 'public-domain',
}
# weakest first, following func_weaker_license in upstream check-copyright
WEAKNESS = ['public-domain', 'FSFULLR', 'LGPL-2.1+', 'LGPL-3+ or GPL-2+', 'LGPL-3+', 'GPL-2+', 'GPL-3+']


def module_licenses():
    """Return {file: set(module license)} from modules/*."""
    res = collections.defaultdict(set)
    for mod in sorted(os.listdir('modules')):
        path = os.path.join('modules', mod)
        if not os.path.isfile(path) or mod in ('README', 'COPYING') or mod.endswith('~'):
            continue
        sections, cur = {}, None
        for line in open(path, encoding='utf-8', errors='replace'):
            m = re.match(r'^([A-Za-z][A-Za-z.-]*):\s*$', line)
            if m:
                cur = m.group(1)
                sections[cur] = []
            elif cur:
                sections[cur].append(line.strip())
        lic = next((l for l in sections.get('License', []) if l), '')
        if mod.startswith('parse-datetime'):
            lic = 'GPL'  # special case in gnulib-tool func_get_license
        lic = lic or 'GPL'
        if lic == 'unmodifiable license text':
            continue
        if lic not in MODULE_LICENSE:
            sys.exit(f'modules/{mod}: unknown License {lic!r}')
        for f in sections.get('Files', []):
            if f:
                res[f].add(MODULE_LICENSE[lic])
    return res


def weakest(lics):
    ranked = sorted(lics, key=lambda l: WEAKNESS.index(l) if l in WEAKNESS else 99)
    a = ranked[0]
    # LGPL-3+ and GPL-2+ are not comparable (upstream: GPLv2+ vs LGPL)
    if a == 'LGPL-3+' and 'GPL-2+' in lics or a == 'GPL-2+' and 'LGPL-3+' in lics:
        return None
    return a


DIR_DEFAULTS = [('doc/', 'GFDL-NIV-1.3+'), ('m4/', 'FSFULLR'), ('modules/', 'FSFAP')]


# --- DEP-5 helpers ---------------------------------------------------------

def read_paragraphs(path):
    text = open(path, encoding='utf-8').read()
    return [p for p in re.split(r'\n\s*\n', text.strip('\n')) if p.strip()]


def field(par, name):
    m = re.search(rf'^{name}:[ \t]*(.*(?:\n[ \t]+.*)*)', par, re.M)
    return m.group(1) if m else None


def dep5_match(pat, f):
    return fnmatch.fnmatchcase(f, pat)


# --- main ------------------------------------------------------------------

def tree_files():
    out = []
    for root, dirs, files in os.walk('.'):
        dirs[:] = sorted(d for d in dirs if not (root == '.' and d in ('.git', '.pc', 'debian')))
        for fn in files:
            p = os.path.normpath(os.path.join(root, fn))
            if not os.path.islink(p):
                out.append(p)
    return sorted(out)


def main():
    check = '--check' in sys.argv
    if not os.path.isdir('modules') or not os.path.isdir(GENDIR):
        sys.exit('run from the top of the gnulib source tree')

    manual = read_paragraphs(f'{GENDIR}/manual.dep5')
    manual_pats = [p for par in manual if field(par, 'Files') for p in field(par, 'Files').split()]
    licenses = read_paragraphs(f'{GENDIR}/licenses.dep5')
    defined = {field(p, 'License').split('\n')[0].strip() for p in licenses}

    modlic = module_licenses()
    info = {}  # file -> (license, holders, source)
    errors = []
    for f in tree_files():
        if any(dep5_match(p, f) for p in manual_pats):
            continue
        with open(f, 'rb') as fh:
            raw = fh.read()
        binary = b'\0' in raw[:HEAD_BYTES]
        full = '' if binary else raw.decode('utf-8', 'replace')
        text = full[:HEAD_BYTES]
        if f.startswith('modules/') and f not in ('modules/README', 'modules/COPYING'):
            text = ''  # descriptions mention licenses; README sets their own
        lic = classify(text) if text else None
        holders = holders_of(text) if text else {}
        src = 'header'
        if lic is None and len(full) > HEAD_BYTES and not f.endswith('.texi'):
            tail = full[-3000:]  # GNU ChangeLog files put the notice at the end
            lic = classify(tail)
            if lic:
                holders = holders_of(tail)
                src = 'trailer'
        if full and not f.startswith('modules/'):
            ex = extras(full)
            for name, hs in ex:
                for h, ys in hs.items():
                    holders.setdefault(h, set()).update(ys)
            ex = [(n, h) for n, h in ex if n != lic]
            if ex:
                names = sorted({n for n, h in ex})
                lic = ' and '.join(([lic] if lic else []) + names)
        if lic is None and holders and re.search(r'licen[cs]e|permission|public domain', text, re.I):
            errors.append(f'{f}: copyright notice without a recognised license')
            continue
        if lic is None and f in modlic:
            lic, src = weakest(modlic[f]), 'module'
            if lic is None:
                errors.append(f'{f}: incomparable module licenses {sorted(modlic[f])}')
                continue
        if lic is None:
            lic = next((l for d, l in DIR_DEFAULTS if f.startswith(d)), DEFAULT)
            src = 'default'
        info[f] = (lic, holders, src)

    if '--list' in sys.argv:
        for f, (lic, holders, src) in sorted(info.items()):
            print(f'{f}\t{lic}\t{src}\t{"; ".join(fmt_copyright(holders))}')
        return

    others = {f for f, (l, h, s) in info.items() if set(h) - {FSF}}

    # Directory globs: walk the tree top-down; a directory gets "dir/*" when
    # the majority license of the FSF-only files below it differs from the
    # inherited one and covers at least half of them.
    below = collections.defaultdict(collections.Counter)
    for f, (lic, h, s) in info.items():
        if f in others:
            continue
        parts = f.split('/')[:-1]
        for i in range(1, len(parts) + 1):
            below['/'.join(parts[:i])][lic] += 1
    subdirs = collections.defaultdict(list)
    for d in below:
        parent = d.rsplit('/', 1)[0] if '/' in d else ''
        subdirs[parent].append(d)
    globs = {}  # dir -> license

    def walk(d, inherited):
        for sd in sorted(subdirs.get(d, [])):
            cnt = below[sd]
            lic, n = cnt.most_common(1)[0]
            total = sum(cnt.values())
            eff = inherited
            if lic != inherited and n * 2 >= total and total >= 3:
                globs[sd] = lic
                eff = lic
            walk(sd, eff)
    walk('', DEFAULT)

    def inherited_of(f):
        parts = f.split('/')[:-1]
        for i in range(len(parts), 0, -1):
            d = '/'.join(parts[:i])
            if d in globs:
                return d
        return ''

    star = collections.defaultdict(lambda: collections.defaultdict(set))  # glob dir -> holders
    explicit = collections.defaultdict(list)   # (license, holders key) -> files
    explicit_h = collections.defaultdict(lambda: collections.defaultdict(set))
    for f, (lic, holders, s) in sorted(info.items()):
        d = inherited_of(f)
        if f not in others and lic == (globs[d] if d else DEFAULT):
            for h, ys in holders.items():
                star[d][h] |= ys
            continue
        key = (lic, tuple(sorted(set(holders) - {FSF})))
        explicit[key].append(f)
        for h, ys in holders.items():
            explicit_h[key][h] |= ys

    used = {DEFAULT} | set(globs.values()) | {k[0] for k in explicit}
    for par in manual:
        l = field(par, 'License')
        if l:
            used.add(l.split('\n')[0].strip())
    for u in sorted(used):
        for part in re.split(r'\s+(?:or|and)\s+', u):
            if part not in defined:
                errors.append(f'license {part!r} has no paragraph in licenses.dep5')
    for d in sorted(defined - {p for u in used for p in re.split(r'\s+(?:or|and)\s+', u)}):
        errors.append(f'license {d!r} in licenses.dep5 is not used')

    if errors and '--list' not in sys.argv:
        print('\n'.join(errors), file=sys.stderr)
        sys.exit(1)

    out = [open(f'{GENDIR}/head.dep5', encoding='utf-8').read().rstrip('\n')]

    def stanza(files, copyright, lic, comment=None):
        s = 'Files:'
        s += (' ' + files[0]) if len(files) == 1 else ''.join('\n ' + x for x in files)
        s += '\nCopyright:'
        s += (' ' + copyright[0]) if len(copyright) == 1 else ''.join('\n ' + x for x in copyright)
        s += f'\nLicense: {lic}'
        if comment:
            s += '\nComment:' + ''.join('\n ' + x if x else '\n .' for x in comment)
        return s

    out.append(stanza(['*'], fmt_copyright(star['']), DEFAULT, [
        'Most files outside the paragraphs below state the GPL-3+ in their',
        'header.  This paragraph also covers the files that have no license',
        'notice, are not listed in a module description and are not under a',
        'directory default: the gnulib manual (section "Copyright") says that',
        'modules are under the GPL unless they say otherwise.']))
    for d in sorted(globs, key=lambda d: (d.count('/'), d)):
        out.append(stanza([d + '/*'], fmt_copyright(star[d]), globs[d]))
    for key in sorted(explicit, key=lambda k: (k[0], k[1])):
        out.append(stanza(explicit[key], fmt_copyright(explicit_h[key]), key[0]))
    out.extend(p.rstrip('\n') for p in manual)
    out.extend(p.rstrip('\n') for p in licenses)
    result = '\n\n'.join(out) + '\n'

    if check:
        cur = open('debian/copyright', encoding='utf-8').read()
        if cur != result:
            sys.exit('debian/copyright is out of date; run debian/copyright-gen/gen-copyright')
        return
    with open('debian/copyright', 'w', encoding='utf-8') as fh:
        fh.write(result)
    src = collections.Counter(s for l, h, s in info.values())
    print(f'{len(info)} files: {dict(src)}; {len(globs)} directory globs, '
          f'{sum(len(v) for v in explicit.values())} files listed in {len(explicit)} paragraphs',
          file=sys.stderr)


if __name__ == '__main__':
    main()
