"""Fictitious paths, real Google robots.txt matcher; no live crawler.
Usage: python3 chemins_robots.py /absolute/path/to/robots_matcher
Build official google/robotstxt commit22b355ff855419e6a3ff8ff09c0ad7fdb17116f9.
URL inputs are ASCII URI strings with uppercase percent triplets. This is
not a general URL normalizer, a crawler, or a universal robots implementation.
"""
from pathlib import Path
from tempfile import TemporaryDirectory
from urllib.parse import quote, unquote
import hashlib,json,subprocess,sys
BINARY=Path(sys.argv[1]).resolve()
POLICIES={
 'encoded': ['/catalogue/%C3%A9t%C3%A9/', '/catalogue/A%2FB/', '/catalogue/promo%2A/'],
 'utf8': ['/catalogue/été/', '/catalogue/A%2FB/', '/catalogue/promo%2A/'],
}
POLICIES['double']= [quote(s,safe='/') for s in POLICIES['encoded']]
POLICIES['decoded']= [unquote(s) for s in POLICIES['encoded']]
CASES=[
 ('accent','/catalogue/%C3%A9t%C3%A9/lampe',False),
 ('sans_accent','/catalogue/ete/lampe',True),
 ('majuscule','/catalogue/%C3%89t%C3%A9/lampe',True),
 ('accent_double','/catalogue/%25C3%25A9t%25C3%25A9/lampe',True),
 ('slash_encode','/catalogue/A%2FB/lampe',False),
 ('deux_segments','/catalogue/A/B/lampe',True),
 ('slash_double','/catalogue/A%252FB/lampe',True),
 ('etoile_litterale','/catalogue/promo%2A/lampe',False),
 ('promo_voisine','/catalogue/promoVIP/lampe',True),
 ('etoile_double','/catalogue/promo%252A/lampe',True),
 ('catalogue','/catalogue/',True),
 ('autre_repertoire','/conseils/%C3%A9t%C3%A9/lampe',True),
]
rows=[]
with TemporaryDirectory(prefix='dawap-robots-encoding-') as td:
 for policy,paths in POLICIES.items():
  f=Path(td)/(policy+'.txt');f.write_text('User-agent: *\n'+''.join('Disallow: '+s+'\n' for s in paths),encoding='utf-8')
  for name,path,want in CASES:
   url='https://example.com'+path
   assert url.isascii() and ' ' not in url
   r=subprocess.run([str(BINARY),str(f),'Googlebot',url],capture_output=True,text=True,check=False)
   assert r.returncode in (0,1),r.stderr
   allowed=r.returncode==0
   assert ('DISALLOWED' if not allowed else ': ALLOWED') in r.stdout
   rows.append({'policy':policy,'case':name,'uri':url,'expectedAllowed':want,'allowed':allowed,'match':allowed==want})
for p in ['encoded','utf8']:
 assert all(x['match'] for x in rows if x['policy']==p)
fail={p:[x['case'] for x in rows if x['policy']==p and not x['match']] for p in POLICIES}
assert fail['double']==['accent','accent_double','slash_encode','slash_double','etoile_litterale','etoile_double'],fail
assert fail['decoded']==['slash_encode','deux_segments','promo_voisine','etoile_double'],fail
print(json.dumps({'sourceCommit':'22b355ff855419e6a3ff8ff09c0ad7fdb17116f9','binarySha256':hashlib.sha256(BINARY.read_bytes()).hexdigest(),'calls':len(rows),'cases':len(CASES),'policies':POLICIES,'failures':fail,'rows':rows,'networkCalls':0,'liveGooglebot':False},ensure_ascii=False,indent=2))