import requests
import re
import html
import json
import jieba
from urllib.parse import urlparse
class myfunsion:
    def a():
        print("test")

class det:#测试url类型
    def image(url):
        style_type_list=[".jpg",".jpeg",".png",".gif",".svg"]
        #将url转小写
        url=url.lower()
        for i in style_type_list:
            if i in url:
                return True
        return False

    def style(url):
        style_type_list=[".js",".css"]
        url=url.lower()
        for i in style_type_list:
            if i in url:
                return True
        return False

    #检查是否有重复
    # arr 数组
    # i   数据
    def repeat(arr,i):
        b=0
        for a in arr:
            if a==i:
                return b
            b+=1
        return -1

    #检测url域名
    def domin(url):
        domin=urlparse(url)
        return domin

class net:
    headers = {
    'user-agent': 'my-app/0.0.1',
    'Content-Type': 'text/html; charset=utf-8'
    }
    def get_html(url):      
        return_js={}
        return_js['url']=url
        try :
            r = requests.get(url,headers=net.headers,verify=False,timeout=(60, 60))
            return_js['status']=r.status_code
            get_content_type='text/html'
            content_type=re.findall(get_content_type,r.headers['Content-Type'])
            if(len(content_type)==0):     
                return_js['contenttype']=r.headers['Content-Type']
            else:
                return_js['contenttype']=get_content_type
            
            if(return_js['contenttype']=='text/html'):
                get_gb2312='gb2312'
                charset=re.findall(get_gb2312,r.headers['Content-Type'])
                if(len(charset)==0):     
                    return_js['charset_type']='utf-8'
                else:
                    return_js['charset_type']='gb2312' 
                htmlx_byt=r.content
                htmlx=str(htmlx_byt,return_js['charset_type'])
                return_js['html']=htmlx
        except:
            return_js['status']=0

        return return_js

    def ana_html(webdata):
        return_js={}
        return_js['status']=webdata['status']
        return_js['url']=webdata['url']
        if webdata['status']==0:
            return_js['score']="0"
            return_js['contenttype']="none"
            return return_js
        return_js['contenttype']=webdata['contenttype']
        #return_js['charset_type']=webdata['charset_type']
        if(return_js['contenttype']=='text/html'):
            thisurlscore=100
            htmlx=webdata['html']    
            ur=urlparse(return_js['url'])
            domain=ur.netloc
            return_js['domain']=domain

            
            #根url
            if(ur.path==''):
                thisurlscore+=50
            if(ur.path=='/'):
                thisurlscore+=50
            #是否带参数
            if(ur.query!=''):
                thisurlscore-=10
            #是否带位置
            if(ur.fragment!=''):
                thisurlscore-=50
            #获取js
            get_htmlx_js='<script[\s\S]*?</script>'
            htmlx_js=re.findall(get_htmlx_js,htmlx)
            #清除htmlx js
            for i in htmlx_js:
                htmlx=htmlx.replace(i,"")

            htmlx_kuaizhao=html.escape(htmlx) #干掉js后截取快照
            htmlx_yasuo=["\n","\r","\t","  "]
            for i in htmlx_yasuo:
                htmlx_kuaizhao=htmlx_kuaizhao.replace(i,"")#压缩一下

            #获取style
            get_htmlx_style='<style[\s\S]*?</style>'
            htmlx_style=re.findall(get_htmlx_style,htmlx)
            #print(htmlx_style)
            #清除htmlx style
            for i in htmlx_style:
                htmlx=htmlx.replace(i,"")

            #获取textarea ,像百度这样的阴间网站有隐藏的输入框里面会有样式什么的奇怪东西
            get_htmlx_textarea='<textarea[\s\S]*?</textarea>'
            htmlx_textarea=re.findall(get_htmlx_textarea,htmlx)
            for i in htmlx_textarea:
                htmlx=htmlx.replace(i,"")
            
            #获取页面中的url并分类
            get_url=r'\"(http[s]?://[\S]*)\"'
            all_url=re.findall(get_url,htmlx)
            style_url_list=[]
            image_url_list=[]
            new_url_list=[]
            for i in all_url:
                if det.style(i):
                    style_url_list.append(i)
                else:
                    if det.image(i):
                        image_url_list.append(i)
                    else:
                        new_url_list.append(i)
            
            #干掉url
            image_url=[]
            new_url=[]
            image_url_js={}
            new_url_js={}
            image_url_int=0
            new_url_int=0
            for i in style_url_list:
                htmlx=htmlx.replace(i,"")
            for i in image_url_list:
                htmlx=htmlx.replace(i,"")
                if det.repeat(image_url,i)==-1:
                    image_url.append(i)
                    image_url_js[str(image_url_int)]=i
                    image_url_int+=1
                    thisurlscore+=2#图片加分
            for i in new_url_list:
                htmlx=htmlx.replace(i,"")
                if det.repeat(new_url,i)==-1:
                    new_url.append(i)
                    new_url_js[str(new_url_int)]=i
                    new_url_int+=1

            #pending
            image_url_str=json.dumps(image_url_js)
            new_url_str=json.dumps(new_url_js)
            return_js['images']=image_url_str
            return_js['newurls']=new_url_str

            #对新url评分
            new_urlscore_js={}
            new_urldomain_js={}
            new_url_int=0
            for i in new_url:
                score=100
                try :
                    te=urlparse(i)

                    #根url
                    if(te.path==''):
                        score+=10
                    if(te.path=='/'):
                        score+=10

                    #是否站内url
                    if(te.netloc==domain):
                        score-=10
                    else:
                        score+=10
                        thisurlscore+=5#站外url加分

                    #是否带参数
                    if(te.query!=''):
                        score-=10

                    #是否带位置
                    if(te.fragment!=''):
                        score-=50
                except:
                    score=0

                new_urlscore_js[str(new_url_int)]=score
                new_urldomain_js[str(new_url_int)]=te.netloc
                new_url_int+=1
            
            new_urlscore_js_str=json.dumps(new_urlscore_js)
            new_urldomain_js_str=json.dumps(new_urldomain_js)
            return_js['newurlscore']=new_urlscore_js_str
            return_js['newurldomain']=new_urldomain_js_str



            #获取标题
            get_htmlx_title='<title[\s\S]*?>([\s\S]*?)</title>'
            htmlx_title=re.findall(get_htmlx_title,htmlx)
            if(len(htmlx_title)>0):
                return_js['title']=htmlx_title[0]
                print(htmlx_title)
            else:
                return_js['title']=return_js['url']

            #get all mate
            get_htmlx_mate_lab='<meta[\S ]*?>'
            htmlx_mate_lab=re.findall(get_htmlx_mate_lab,htmlx)
            htmlx_mate=[]
            htmlx_mate_str=""
            htmlx_mate_js={}
            for i in htmlx_mate_lab:
                get_htmlx_mate_name='name="([\S ]*?)"'
                htmlx_mate_name=re.findall(get_htmlx_mate_name,i)
                if len(htmlx_mate_name)==1:#只允许有1个名字 其他阴间标签不要
                    get_htmlx_mate_val='content="([\S\s]*?)"'
                    htmlx_mate_val=re.findall(get_htmlx_mate_val,i)
                    if len(htmlx_mate_val)==1:
                        if htmlx_mate_name[0]!='':
                            #print(htmlx_mate_name[0])
                            #print(htmlx_mate_val[0])
                            #htmlx_mate_str+="\""+htmlx_mate_name[0]+"\":\""+htmlx_mate_val[0]+"\","
                            htmlx_mate_js[htmlx_mate_name[0]]=htmlx_mate_val[0]
                            htmlx_mate.append(htmlx_mate_val[0])
            htmlx_mate_str=json.dumps(htmlx_mate_js)
            return_js['mate']=htmlx_mate_str

            #获取所有标签内容
            get_htmlx_all_tab='>([\s\S]*?)<'
            htmlx_tab=re.findall(get_htmlx_all_tab,htmlx)
            #print(htmlx_tab)

            #inpute mate
            for i in htmlx_mate:
                htmlx_tab.append(i)

            #先干掉\r\n\t 保留其他标点符号做语义识别
            get_text_rnt=['\r','\n','\t','  ']
            htmlx_tab_len=len(htmlx_tab)
            for i in range(htmlx_tab_len-1,-1,-1):#从后往前数
                for t in get_text_rnt:htmlx_tab[i]=htmlx_tab[i].replace(t,"")
                htmlx_tab[i]=htmlx_tab[i].strip()
                if htmlx_tab[i]=='':del htmlx_tab[i]#删除空
            #print(htmlx_tab)

            get_htmlx_BDFH=["\"","\\","^","’","=","/","、","“","”","#","©","|","_","-"," ","*",";","&","$","%","!","?",",",".","(",")","[","]","{","}","<",">","￥","%","！","？","，","。","（","）","【","】","《","》","："]
            #提取关键词
            htmlx_sents_js={}
            htmlx_words_js={}
            sents=[]
            words=[]
            words_int=0
            links_js={}
            links_int=0
            sents_int=0
            for i in range(len(htmlx_tab)):
                sent=htmlx_tab[i]              #获取每个句子
                word=jieba.cut_for_search(sent)#从每个句子获取单词
                #数据库限制每个句子256字符
                sent=sent[0:256]
                if det.repeat(sents,sent)==-1: #查重
                    sents.append(sent)
                    htmlx_sents_js[str(sents_int)]=sent
                    
                    for t in word:
                        #删除一些意义不大的符号
                        #数据库限制每个单词8字符
                        w=t[0:8]
                        for n in get_htmlx_BDFH:
                            w=w.replace(n,"")
                        if w!='':
                            words_det_int=-1
                            words_det_int=det.repeat(words,w)
                            if words_det_int==-1:
                                words.append(w)                       
                                words_det_int=words_int
                                htmlx_words_js[str(words_int)]=w
                                words_int+=1
                            links_js[str(links_int)]=str(words_det_int)+":"+str(sents_int)
                            links_int+=1
                    sents_int+=1
                else:
                    #重复的内容
                    thisurlscore-=1

            return_js['sents_int']=sents_int
            return_js['words_int']=words_int
            return_js['links_int']=links_int            

            htmlx_sents_str=json.dumps(htmlx_sents_js)
            htmlx_words_str=json.dumps(htmlx_words_js)
            links_str=json.dumps(links_js)

            return_js['sents']=htmlx_sents_str
            return_js['words']=htmlx_words_str
            return_js['links']=links_str

            return_js['htmlx']=htmlx_kuaizhao
            return_js['score']=thisurlscore#对当前url评分
        else:
            return_js['title']=return_js['url']
        return return_js








test=0

getone={}
getone['password']="(*&RV^*(&VRH*(V)))"
getone_post_url="https://lmve.net/php/getoneurl.php"
post_url="https://lmve.net/php/newurlreptile.php"

if test==0:
    while 1:
        print("**本地消息**\n获取url..")
        try:
            r = requests.post(getone_post_url,data=getone)
            if r.status_code==200:       
                html_byt=r.content
                html_text=str(html_byt,'utf-8')
                print("获取url成功:")
                print(html_text)
                print("\n")
                r.close()
                if html_text!='':
                    print("获取text..")
                    text=net.get_html(html_text)
                    print("开始分析..")
                    postdata=net.ana_html(text)                   
                    print("分析完成")
                    postdata['password']="(*&RV^*(&VRH*(V)))"
                    if postdata['contenttype']=='text/html':
                        print("分离数据成功")
                        print("url:")
                        print(postdata['url'])
                        print("status code:")
                        print(postdata['status'])
                        print("content type:")
                        print(postdata['contenttype'])
                        print("Title:")
                        print(postdata['title'])
                        print("Sents const:")
                        print(postdata['sents_int'])
                        print("Words const:")
                        print(postdata['words_int'])
                        print("Links const:")
                        print(postdata['links_int'])
                    else :
                        print("分离数据成功 no html")
                        print("url:")
                        print(postdata['url'])
                        print("status code:")
                        print(postdata['status'])
                        print("content type:")
                        print(postdata['contenttype'])
                    print("开始发送..")
                    try:
                        r1 = requests.post(post_url,data=postdata)
                        if r1.status_code==200:       
                            html_byt=r1.content
                            html_text=str(html_byt,'utf-8')
                            print("发送成功:200")
                            print(html_text)
                            r1.close()
                        else:
                            print("error"+str(r1.status_code))            
                    except:
                        print("try aga")
            else:
                print("获取url失败")
        except:
            print("tyr agin")
            
       



if test==1:
    postdata=net.ana_html(net.get_html("https://www.foreverblog.cn/blog/1958.html"))
    postdata['password']="(*&RV^*(&VRH*(V)))"   
    r1 = requests.post(post_url,data=postdata)
    if r1.status_code==200:       
        html_byt=r1.content
        html_text=str(html_byt,'utf-8')
        print("发送成功:200\n")
        print(html_text)
        r1.close()
    else:
        print("error"+str(r1.status_code))