#!/usr/bin/env python3
"""Download the public bibliographic CSV without an account or third-party packages."""
import argparse
import csv
import hashlib
import json
import io
from collections import Counter
from datetime import datetime, timezone
import os
import tempfile
import time
from urllib.error import HTTPError
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import Request, urlopen


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--compound', choices=['psilocybin', 'psilocin'], default='psilocybin')
    parser.add_argument('--output', type=Path, default=Path('psilocybin-papers.csv'))
    parser.add_argument('--limit', type=int, default=50000)
    args = parser.parse_args()
    if not 1 <= args.limit <= 50000:
        parser.error('--limit must be between 1 and 50000')
    if args.output.exists():
        parser.error('Output already exists; choose another filename to preserve your previous input')
    manifest = args.output.with_suffix(args.output.suffix + '.manifest.json')
    if manifest.exists():
        parser.error('Provenance manifest already exists; choose another output filename')
    url = 'https://psilocybin-research.com/api/v1/papers?' + urlencode({'format': 'csv', 'compound': args.compound, 'limit': args.limit})
    request = Request(url, headers={'User-Agent': 'PsilocybinResearchClient/1.0', 'Accept': 'text/csv'})
    temporary = None
    response = None
    for attempt in range(4):
        try:
            response = urlopen(request, timeout=90)
            break
        except HTTPError as error:
            if error.code != 429 or attempt == 3:
                raise
            delay = error.headers.get("Retry-After", "5")
            time.sleep(min(60, max(1, int(delay) if delay.isdigit() else 5)))
    try:
        with response:
            if response.headers.get_content_type() != 'text/csv':
                raise RuntimeError('The server did not return CSV; no output was saved')
            with tempfile.NamedTemporaryFile(dir=args.output.parent, delete=False) as target:
                temporary = target.name
                while chunk := response.read(65536):
                    target.write(chunk)
            raw=Path(temporary).read_bytes()
            reader=csv.DictReader(io.StringIO(raw.decode('utf-8-sig')))
            records=list(reader)
            if not {'id','title','doi','publication_status'}.issubset(reader.fieldnames or []):
                raise RuntimeError('CSV has no usable records or unexpected columns; no output saved')
            provenance={'retrieved_at_utc':datetime.now(timezone.utc).isoformat(),'source_url':url,'sha256':hashlib.sha256(raw).hexdigest(),'records':len(records),'publication_status_counts':dict(Counter(row['publication_status'] or 'not reported' for row in records)),'limit':args.limit,'possibly_truncated':len(records)>=args.limit,'interpretation':'Bibliographic records are not independent trials or participants. Preserve statuses, source identifiers and this input checksum. Source coverage is incomplete; a limit reached needs a larger or different retrieval.'}
            # Link is atomic and refuses a file that appeared during the download.
            os.link(temporary, args.output)
            with tempfile.NamedTemporaryFile(dir=args.output.parent,delete=False) as meta:
                meta.write((json.dumps(provenance,indent=2)+'\n').encode());meta_name=meta.name
            try:os.link(meta_name,manifest)
            finally:os.unlink(meta_name)
            print(f'Saved {response.headers.get("X-Publication-Tracker-Export-Count", "available")} records to {args.output}')
            print('Preserve this input and record its retrieval date before analysis.')
    finally:
        if temporary:
            os.unlink(temporary)

if __name__ == '__main__':
    main()