import requests
import socket
import time
import urllib.parse
import random
from bs4 import BeautifulSoup
from fastapi.responses import JSONResponse

async def scrape_url_improved(request):
    """
    改进版本的URL抓取函数，解决服务器部署后的网络连接问题
    """
    try:
        print(f"开始抓取URL: {request.url}")
        
        # 设置请求头，模拟浏览器访问
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Cache-Control': 'max-age=0'
        }
        
        # 添加DNS解析测试
        try:
            url_parts = urllib.parse.urlparse(request.url)
            hostname = url_parts.hostname
            print(f"正在解析域名: {hostname}")
            ip_address = socket.gethostbyname(hostname)
            print(f"域名解析成功: {hostname} -> {ip_address}")
        except socket.gaierror as dns_error:
            print(f"DNS解析失败: {dns_error}")
            return JSONResponse(
                status_code=500,
                content={"error": f"DNS解析失败，无法访问 {hostname}: {str(dns_error)}"},
            )
        
        # 使用重试机制，增加连接和读取超时时间
        max_retries = 3
        timeout_settings = (30, 90)  # (连接超时, 读取超时)
        
        response = None
        last_error = None
        
        for attempt in range(max_retries):
            try:
                print(f"第 {attempt + 1} 次尝试连接...")
                start_time = time.time()
                
                # 发送HTTP请求获取页面内容，增加超时时间和重试
                response = requests.get(
                    request.url, 
                    headers=headers, 
                    timeout=timeout_settings,
                    verify=False,  # 临时禁用SSL验证，避免证书问题
                    allow_redirects=True
                )
                
                elapsed_time = time.time() - start_time
                print(f"请求成功，耗时: {elapsed_time:.2f}秒")
                response.raise_for_status()  # 如果请求失败，抛出异常
                break
                
            except requests.exceptions.Timeout as timeout_error:
                last_error = timeout_error
                print(f"第 {attempt + 1} 次尝试超时: {timeout_error}")
                if attempt < max_retries - 1:
                    wait_time = (attempt + 1) * 2  # 递增等待时间
                    print(f"等待 {wait_time} 秒后重试...")
                    time.sleep(wait_time)
                continue
                
            except requests.exceptions.ConnectionError as conn_error:
                last_error = conn_error
                print(f"第 {attempt + 1} 次尝试连接错误: {conn_error}")
                if attempt < max_retries - 1:
                    wait_time = (attempt + 1) * 2
                    print(f"等待 {wait_time} 秒后重试...")
                    time.sleep(wait_time)
                continue
                
            except requests.exceptions.RequestException as req_error:
                last_error = req_error
                print(f"第 {attempt + 1} 次尝试请求错误: {req_error}")
                if attempt < max_retries - 1:
                    wait_time = (attempt + 1) * 2
                    print(f"等待 {wait_time} 秒后重试...")
                    time.sleep(wait_time)
                continue
        
        # 如果所有重试都失败了
        if response is None:
            error_msg = f"经过 {max_retries} 次重试后仍然无法连接到 {request.url}"
            if last_error:
                error_msg += f"，最后错误: {str(last_error)}"
            print(error_msg)
            return JSONResponse(
                status_code=500,
                content={
                    "error": error_msg,
                    "suggestions": [
                        "检查服务器网络连接",
                        "确认目标网站是否可访问", 
                        "检查防火墙设置",
                        "考虑配置代理服务器",
                        "联系系统管理员检查网络配置"
                    ]
                },
            )
        
        # 设置编码以正确处理中文字符
        response.encoding = 'utf-8'
        
        # 解析HTML
        soup = BeautifulSoup(response.text, 'html.parser')
        
        # 获取基础URL用于解析相对路径
        url_parts = urllib.parse.urlparse(request.url)
        base_url = f"{url_parts.scheme}://{url_parts.netloc}"
        
        # 初始化数据字典
        teacher_data = {
            "id": f"BLG{random.randint(10000, 99999)}",
            "photo": f"data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='100' height='120' viewBox='0 0 100 120'%3E%3Crect width='100' height='120' fill='%234986ff' opacity='0.3'/%3E%3Ccircle cx='50' cy='45' r='25' fill='%234986ff' opacity='0.6'/%3E%3Ccircle cx='50' cy='95' r='35' fill='%234986ff' opacity='0.6'/%3E%3C/svg%3E",
            "evaluationData": [
                round(min(100, max(60, 70 + 20 * (0.5 - random.random())))) for _ in range(6)
            ]
        }
        
        # 从教师信息表提取基本信息
        info_table = soup.find('div', class_='wz_teacher')
        if info_table:
            table = info_table.find('table')
            if table:
                rows = table.find_all('tr')
                
                # 提取姓名、性别、出生年月
                if len(rows) > 0:
                    cells = rows[0].find_all('td')
                    if len(cells) >= 6:
                        teacher_data["name"] = cells[1].text.strip()
                        teacher_data["gender"] = cells[3].text.strip()
                        teacher_data["birthDate"] = cells[5].text.strip()
                
                # 提取职称、职务、最高学历
                if len(rows) > 1:
                    cells = rows[1].find_all('td')
                    if len(cells) >= 6:
                        teacher_data["title"] = cells[1].text.strip()
                        position = cells[3].text.strip()
                        teacher_data["position"] = position if position else ""
                        teacher_data["education"] = cells[5].text.strip()
                
                # 提取学科方向
                if len(rows) > 2:
                    cells = rows[2].find_all('td')
                    if len(cells) >= 2:
                        teacher_data["academicDirection"] = cells[1].text.strip()
                
                # 提取人才计划和办公地点
                if len(rows) > 3:
                    cells = rows[3].find_all('td')
                    if len(cells) >= 6:
                        talent_plan = cells[1].text.strip()
                        teacher_data["talentPlan"] = talent_plan if talent_plan else ""
                        teacher_data["officeLocation"] = cells[5].text.strip()
                
                # 提取电子邮件和联系方式
                if len(rows) > 4:
                    cells = rows[4].find_all('td')
                    if len(cells) >= 6:
                        email = cells[1].text.strip()
                        teacher_data["email"] = email if email else ""
                        phone = cells[5].text.strip()
                        teacher_data["phone"] = phone if phone else ""
                
                # 提取通讯地址
                if len(rows) > 5:
                    cells = rows[5].find_all('td')
                    if len(cells) >= 2:
                        teacher_data["address"] = cells[1].text.strip()
                
                # 提取导师类型
                if len(rows) > 6:
                    cells = rows[6].find_all('td')
                    if len(cells) >= 2:
                        teacher_data["tutorType"] = cells[1].text.strip()
        
        # 提取照片
        photo_element = soup.select_one('.teacherInfo .img img')
        if photo_element and photo_element.get('src'):
            img_src = photo_element['src']
            
            # 处理相对路径，构建完整的图片URL
            if img_src.startswith('../../../'):
                # 从URL获取基础路径（移除文件名和最后两级目录）
                url_parts = request.url.split('/')
                if len(url_parts) >= 4:
                    base_path = '/'.join(url_parts[:-3])
                    img_url = f"{base_path}/{img_src[9:]}"  # 移除 '../../../'
                else:
                    img_url = urllib.parse.urljoin(base_url, img_src)
            else:
                img_url = urllib.parse.urljoin(base_url, img_src)
            
            # 直接保存完整的图片URL，不下载到本地
            teacher_data["photo"] = img_url
        
        # 提取详细信息部分
        content_divs = soup.select('.con01_t')
        for div in content_divs:
            heading = div.find('h3')
            if not heading:
                continue
                
            heading_text = heading.text.strip()
            
            # 获取该部分的所有段落文本
            paragraphs = [p.text.strip() for p in div.find_all('p') if p.text.strip()]
            section_content = '\n'.join(paragraphs)
            
            # 根据标题将内容映射到相应字段
            if '教育与工作经历' in heading_text:
                teacher_data["eduWorkHistory"] = section_content
            elif '研究方向' in heading_text:
                teacher_data["researchDirection"] = section_content
            elif '近5年承担的科研项目' in heading_text or '近五年承担的科研项目' in heading_text:
                teacher_data["recentProjects"] = section_content
                # 计算项目数量
                project_count = len([p for p in paragraphs if p.strip().startswith(str(len(paragraphs) - paragraphs.index(p))+".")])
                if project_count > 0:
                    teacher_data["projects"] = f"{project_count}项"
                else:
                    teacher_data["projects"] = f"{len(paragraphs)}项"
            elif '代表性学术论文' in heading_text:
                teacher_data["representativePapers"] = section_content
                # 计算论文数量
                paper_count = len([p for p in paragraphs if p.strip().startswith("[")])
                if paper_count > 0:
                    teacher_data["papers"] = f"{paper_count}篇"
                else:
                    teacher_data["papers"] = f"{len(paragraphs)}篇"
            elif '授权国家发明专利' in heading_text or '专利' in heading_text:
                teacher_data["patents"] = section_content
        
        print(f"抓取成功，提取到教师数据: {teacher_data.get('name', '未知')}")
        return teacher_data
        
    except Exception as e:
        print(f"抓取错误: {str(e)}")
        return JSONResponse(
            status_code=500,
            content={"error": f"抓取网页失败: {str(e)}"},
        ) 