import json,subprocess,collections,time,os
UA="Aethos-Agentic-Probe/1.0 (+https://aethoshq.com/agent-probe; contact: hello@aethoshq.com)"
PROFILE="https://aethoshq.com/.well-known/ucp-agent-profile.json"
QUERIES=["running shoes","face moisturizer","dog collar","standing desk","coffee beans",
         "yoga mat","leather wallet","cast iron skillet","reading glasses","protein powder",
         "scented candle","mountain bike helmet"]
def search(q,limit=50):
    body=json.dumps({"jsonrpc":"2.0","method":"tools/call","id":1,"params":{"name":"search_catalog","arguments":{
        "meta":{"ucp-agent":{"profile":PROFILE}},
        "catalog":{"query":q,"pagination":{"limit":limit}}}}})
    r=subprocess.run(["curl","-sS","-m","60","-X","POST","https://catalog.shopify.com/api/ucp/mcp",
        "-H","Content-Type: application/json","-H","Accept: application/json, text/event-stream",
        "-H",f"User-Agent: {UA}","-d",body],capture_output=True,text=True)
    return r.stdout
def cnt(v):
    if v is None: return None
    if isinstance(v,list): return len(v)
    if isinstance(v,str): return len([p for p in v.split('\n') if p.strip()])
    return -1
rows=[]
for q in QUERIES:
    fn=f"raw/{q.replace(' ','-')}.json"
    if os.path.exists(fn): raw=open(fn).read()
    else:
        raw=search(q); open(fn,'w').write(raw); time.sleep(3)
    try: d=json.loads(raw)
    except Exception: print(q,"PARSE FAIL",raw[:120]); continue
    prods=d.get('result',{}).get('structuredContent',{}).get('products',[])
    for p in prods:
        m=p.get('metadata') or {}
        seller=None
        for v in p.get('variants',[]):
            s=v.get('seller')
            if isinstance(s,dict): seller=s.get('domain') or s.get('name'); break
        rows.append({'q':q,'id':p.get('id'),'title':p.get('title'),'seller':seller,
            'usp':cnt(m.get('unique_selling_points')),'usp_t':type(m.get('unique_selling_points')).__name__,
            'feat':cnt(m.get('top_features')),'feat_t':type(m.get('top_features')).__name__,
            'spec':cnt(m.get('tech_specs')),'spec_t':type(m.get('tech_specs')).__name__,
            'meta_keys':sorted(m.keys())})
    print(f"{q!r}: {len(prods)}")
json.dump(rows,open('dataset.json','w'),indent=1)
uniq={r['id']:r for r in rows}
rows=list(uniq.values())
print()
print("UNIQUE PRODUCTS:",len(rows))
print("DISTINCT SELLER STORES:",len({r['seller'] for r in rows if r['seller']}))
print("QUERIES:",len(QUERIES))
print()
print("usp count dist :",dict(sorted(collections.Counter(r['usp'] for r in rows).items(),key=lambda x:(x[0] is None,x[0]))))
print("usp type dist  :",dict(collections.Counter(r['usp_t'] for r in rows)))
print("feat count dist:",dict(sorted(collections.Counter(r['feat'] for r in rows).items(),key=lambda x:(x[0] is None,x[0]))))
print("feat type dist :",dict(collections.Counter(r['feat_t'] for r in rows)))
print("spec count dist:",dict(sorted(collections.Counter(r['spec'] for r in rows).items(),key=lambda x:(x[0] is None,x[0]))))
print("spec type dist :",dict(collections.Counter(r['spec_t'] for r in rows)))
print()
print("metadata key sets:",collections.Counter(tuple(r['meta_keys']) for r in rows))
print()
print("NON-1 USP:",[ (r['seller'],r['title'],r['usp']) for r in rows if r['usp']!=1 ][:20])