#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Search Quality Analysis - Detailed comparison of search result ranking and relevance This script analyzes the quality of search results by comparing: 1. Result ranking changes 2. Keyword matching quality 3. Field weight impact 4. Synonym expansion effectiveness Usage: python3 analyze_search_quality.py [--query "btn primary"] [--domain component] """ import argparse from core import search, CSV_CONFIG def analyze_ranking_quality(query, domain, max_results=10): """详细分析搜索结果的质量""" # 运行四种配置 configs = { "标准BM25": {"use_weights": False, "expand_synonyms": False}, "仅同义词扩展": {"use_weights": False, "expand_synonyms": True}, "仅字段权重": {"use_weights": True, "expand_synonyms": False}, "完整优化": {"use_weights": True, "expand_synonyms": True}, } results = {} for config_name, config in configs.items(): result = search(query, domain, max_results, **config, use_cache=False) results[config_name] = result.get("results", []) # 分析每个结果在不同配置下的排名 analysis = { "query": query, "domain": domain, "configs": {}, "ranking_changes": [], "keyword_matches": {}, "summary": {} } # 收集所有唯一结果 all_results = {} for config_name, result_list in results.items(): for rank, item in enumerate(result_list, 1): key = item.get("Component") or item.get("Element") or item.get("Pattern") or item.get("Icon Name", "") if key not in all_results: all_results[key] = { "item": item, "rankings": {} } all_results[key]["rankings"][config_name] = rank # 分析每个配置的结果 for config_name, result_list in results.items(): keywords_in_top3 = [] for item in result_list[:3]: keywords = item.get("Keywords", "") component = item.get("Component") or item.get("Element") or "N/A" keywords_in_top3.append(f"{component}: {keywords}") analysis["configs"][config_name] = { "count": len(result_list), "top3": [item.get("Component") or item.get("Element") or item.get("Pattern") or "N/A" for item in result_list[:3]], "top3_keywords": keywords_in_top3 } # 计算排名变化 baseline_top3 = set(analysis["configs"]["标准BM25"]["top3"]) optimized_top3 = set(analysis["configs"]["完整优化"]["top3"]) analysis["ranking_changes"] = { "新增到Top3": list(optimized_top3 - baseline_top3), "移出Top3": list(baseline_top3 - optimized_top3), "排名提升": [], "排名下降": [] } # 分析排名变化详情 for key, data in all_results.items(): baseline_rank = data["rankings"].get("标准BM25") optimized_rank = data["rankings"].get("完整优化") if baseline_rank and optimized_rank: change = baseline_rank - optimized_rank # 正数表示排名提升 if change > 0: analysis["ranking_changes"]["排名提升"].append({ "component": key, "from": baseline_rank, "to": optimized_rank, "improvement": change }) elif change < 0: analysis["ranking_changes"]["排名下降"].append({ "component": key, "from": baseline_rank, "to": optimized_rank, "degradation": abs(change) }) # 关键词匹配质量分析 query_tokens = set(query.lower().split()) analysis["keyword_matches"] = {} for config_name, result_list in results.items(): matches = [] for item in result_list[:5]: keywords = item.get("Keywords", "").lower() component = item.get("Component") or item.get("Element") or "N/A" # 检查关键词匹配 matched_tokens = [token for token in query_tokens if token in keywords] match_score = len(matched_tokens) / len(query_tokens) if query_tokens else 0 matches.append({ "component": component, "keywords": item.get("Keywords", ""), "matched_tokens": matched_tokens, "match_score": match_score }) analysis["keyword_matches"][config_name] = matches # 生成总结 analysis["summary"] = { "结果数量": { "标准BM25": analysis["configs"]["标准BM25"]["count"], "完整优化": analysis["configs"]["完整优化"]["count"], }, "排名提升项数": len(analysis["ranking_changes"]["排名提升"]), "Top3变化": len(analysis["ranking_changes"]["新增到Top3"]), "关键词匹配质量": { "标准BM25_avg": sum(m["match_score"] for m in analysis["keyword_matches"]["标准BM25"][:3]) / 3 if analysis["keyword_matches"]["标准BM25"] else 0, "完整优化_avg": sum(m["match_score"] for m in analysis["keyword_matches"]["完整优化"][:3]) / 3 if analysis["keyword_matches"]["完整优化"] else 0, } } return analysis, results def format_analysis_report(analysis, results): """格式化分析报告""" lines = [] lines.append("=" * 80) lines.append("# 🔍 搜索结果质量分析报告") lines.append("=" * 80) lines.append("") lines.append(f"**查询**: `{analysis['query']}`") lines.append(f"**Domain**: `{analysis['domain']}`") lines.append("") # 总结 lines.append("## 📊 总结") summary = analysis["summary"] lines.append(f"- **结果数量**: 标准BM25={summary['结果数量']['标准BM25']}, 完整优化={summary['结果数量']['完整优化']}") lines.append(f"- **排名提升项数**: {summary['排名提升项数']}") lines.append(f"- **Top3新增项**: {len(analysis['ranking_changes']['新增到Top3'])}") lines.append(f"- **关键词匹配质量**: 标准BM25={summary['关键词匹配质量']['标准BM25_avg']:.2%}, 完整优化={summary['关键词匹配质量']['完整优化_avg']:.2%}") lines.append("") # Top 3 对比 lines.append("## 🎯 Top 3 结果对比") for config_name in ["标准BM25", "仅同义词扩展", "仅字段权重", "完整优化"]: top3 = analysis["configs"][config_name]["top3"] lines.append(f"### {config_name}") for i, item in enumerate(top3, 1): lines.append(f" {i}. {item}") lines.append("") # 排名变化 lines.append("## 📈 排名变化分析") changes = analysis["ranking_changes"] if changes["新增到Top3"]: lines.append("### ✅ 新增到Top3") for item in changes["新增到Top3"]: lines.append(f" - {item}") lines.append("") if changes["排名提升"]: lines.append("### ⬆️ 排名提升") for item in sorted(changes["排名提升"], key=lambda x: x["improvement"], reverse=True): lines.append(f" - **{item['component']}**: {item['from']} → {item['to']} (提升 {item['improvement']} 位)") lines.append("") if changes["排名下降"]: lines.append("### ⬇️ 排名下降") for item in sorted(changes["排名下降"], key=lambda x: x["degradation"], reverse=True): lines.append(f" - **{item['component']}**: {item['from']} → {item['to']} (下降 {item['degradation']} 位)") lines.append("") # 关键词匹配质量 lines.append("## 🔑 关键词匹配质量 (Top 5)") query_tokens = set(analysis["query"].lower().split()) lines.append(f"**查询词**: {', '.join(query_tokens)}") lines.append("") for config_name in ["标准BM25", "完整优化"]: lines.append(f"### {config_name}") matches = analysis["keyword_matches"][config_name][:5] for i, match in enumerate(matches, 1): matched = match["matched_tokens"] unmatched = query_tokens - set(matched) lines.append(f" {i}. **{match['component']}**") lines.append(f" - 匹配: {', '.join(matched) if matched else '无'}") lines.append(f" - 未匹配: {', '.join(unmatched) if unmatched else '无'}") lines.append(f" - 匹配度: {match['match_score']:.2%}") lines.append("") # 功能效果分析 lines.append("## ⚙️ 各优化功能效果分析") # 同义词扩展效果 baseline_top3 = set(analysis["configs"]["标准BM25"]["top3"]) synonym_top3 = set(analysis["configs"]["仅同义词扩展"]["top3"]) synonym_new = synonym_top3 - baseline_top3 if synonym_new: lines.append("### 同义词扩展") lines.append(" - ✅ 有效：改变了Top3结果") for item in synonym_new: lines.append(f" - 新增: {item}") else: lines.append("### 同义词扩展") lines.append(" - ℹ️ 未改变Top3，但可能改善了排序") lines.append("") # 字段权重效果 weight_top3 = set(analysis["configs"]["仅字段权重"]["top3"]) weight_new = weight_top3 - baseline_top3 if weight_new: lines.append("### 字段权重") lines.append(" - ✅ 有效：改变了Top3结果") for item in weight_new: lines.append(f" - 新增: {item}") else: lines.append("### 字段权重") lines.append(" - ℹ️ 未改变Top3，但可能改善了排序") lines.append("") lines.append("=" * 80) return "\n".join(lines) def main(): parser = argparse.ArgumentParser( description="Search Quality Analysis - Detailed comparison of search result ranking", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: python3 analyze_search_quality.py --query "btn primary" --domain component python3 analyze_search_quality.py --query "hover cursor" --domain component --output analysis.md """ ) parser.add_argument("--query", "-q", default="btn primary", help="Search query") parser.add_argument("--domain", "-d", default="component", help="Search domain") parser.add_argument("--max-results", "-n", type=int, default=10, help="Max results to analyze") parser.add_argument("--output", "-o", help="Output file path (default: print to console)") args = parser.parse_args() print(f"🔍 分析查询: '{args.query}' (domain: {args.domain})") print("正在运行分析...") print() analysis, results = analyze_ranking_quality(args.query, args.domain, args.max_results) report = format_analysis_report(analysis, results) if args.output: with open(args.output, 'w', encoding='utf-8') as f: f.write(report) print(f"📄 分析报告已保存到: {args.output}") else: print(report) if __name__ == "__main__": main()