# encoding:utf-8
#简易爬虫

#     requests  用于下载网页源码  第三方库 通过pip安装  
#     re        正则表达式要用到
#     threading 启用多线程
#     os，sys   用于创建目录
#     time      用于获得当天时间
from asyncio.windows_events import NULL
import requests
import re
import jieba
import html
import json

#from bs4 import BeautifulSoup as bs
#from lxml import etree

def det_image(url):
    style_type_list=[".jpg",".jpeg",".png",".gif"]
    #将url转小写
    url=url.lower()
    for i in style_type_list:
        if i in url:
            return True
    return False

def det_style(url):
    style_type_list=[".js",".css"]
    url=url.lower()
    for i in style_type_list:
        if i in url:
            return True
    return False

def det_repeat(arr,i):
    b=0
    for a in arr:
        if a==i:
            return b
        b+=1
    return -1

def get_page_words(url):

    retunrn_js={}
    begin_url=url
    headers = {
        'user-agent': 'my-app/0.0.1',
        'Content-Type': 'text/html; charset=utf-8'
        }

    retunrn_js['url']=begin_url
    #print(begin_url)
    #获取整个页面
    try:
        r = requests.get(begin_url,headers=headers,verify=False)
        retunrn_js['status']=r.status_code
        get_content_type='text/html'
        content_type=re.findall(get_content_type,r.headers['Content-Type'])
        if(len(content_type)==0):     
            retunrn_js['contenttype']=r.headers['Content-Type']
        else:
            retunrn_js['contenttype']=get_content_type

        get_gb2312='gb2312'
        charset=re.findall(get_gb2312,r.headers['Content-Type'])
        if(len(charset)==0):     
            charset_type='utf-8'
        else:
            charset_type='gb2312'    

        if(retunrn_js['contenttype']=='text/html'):

            htmlx_byt=r.content
            htmlx=str(htmlx_byt,charset_type)
            #print(htmlx)

            #获取js
            get_htmlx_js='<script[\s\S]*?</script>'
            htmlx_js=re.findall(get_htmlx_js,htmlx)
            #清除htmlx js
            for i in htmlx_js:
                htmlx=htmlx.replace(i,"")

            htmlx_kuaizhao=html.escape(htmlx) #干掉js后截取快照
            htmlx_yasuo=["\n","\r","\t","  "]
            for i in htmlx_yasuo:
                htmlx_kuaizhao=htmlx_kuaizhao.replace(i,"")#压缩一下

            #获取style
            get_htmlx_style='<style[\s\S]*?</style>'
            htmlx_style=re.findall(get_htmlx_style,htmlx)
            #print(htmlx_style)
            #清除htmlx style
            for i in htmlx_style:
                htmlx=htmlx.replace(i,"")

            #print(htmlx)

            #获取textarea ,像百度这样的阴间网站有隐藏的输入框里面会有样式什么的奇怪东西
            get_htmlx_textarea='<textarea[\s\S]*?</textarea>'
            htmlx_textarea=re.findall(get_htmlx_textarea,htmlx)
            for i in htmlx_textarea:
                htmlx=htmlx.replace(i,"")

            #获取页面中的url并分类
            get_url=r'"(http[s]?://[\S]*)"'
            all_url=re.findall(get_url,htmlx)
            style_url_list=[]
            image_url_list=[]
            new_url_list=[]
            for i in all_url:
                if det_style(i):
                    style_url_list.append(i)
                else:
                    if det_image(i):
                        image_url_list.append(i)
                    else:
                        new_url_list.append(i)

            #干掉url
            image_url=[]
            new_url=[]
            image_url_js={}
            new_url_js={}
            image_url_int=0
            new_url_int=0
            for i in style_url_list:
                htmlx=htmlx.replace(i,"")
            for i in image_url_list:
                htmlx=htmlx.replace(i,"")
                if det_repeat(image_url,i)==-1:
                    image_url.append(i)
                    image_url_js[str(image_url_int)]=i
                    image_url_int+=1
            for i in new_url_list:
                htmlx=htmlx.replace(i,"")
                if det_repeat(new_url,i)==-1:
                    new_url.append(i)
                    new_url_js[str(new_url_int)]=i
                    new_url_int+=1

            image_url_str=json.dumps(image_url_js)
            new_url_str=json.dumps(new_url_js)

            #获取标签内容
            #soup=bs(htmlx,'htmlx.parser') 
            #dom = etree.htmlx(htmlx)
            #print(soup.body)

            #获取标题
            get_htmlx_title='<title[\s\S]*?>([\s\S]*?)</title>'
            htmlx_title=re.findall(get_htmlx_title,htmlx)
            #print(htmlx_title)

            #get all mate
            get_htmlx_mate_lab='<meta[\S ]*?>'
            htmlx_mate_lab=re.findall(get_htmlx_mate_lab,htmlx)
            htmlx_mate=[]
            htmlx_mate_str=""
            htmlx_mate_js={}
            for i in htmlx_mate_lab:
                get_htmlx_mate_name='name="([\S ]*?)"'
                htmlx_mate_name=re.findall(get_htmlx_mate_name,i)
                if len(htmlx_mate_name)==1:#只允许有1个名字 其他阴间标签不要
                    get_htmlx_mate_val='content="([\S\s]*?)"'
                    htmlx_mate_val=re.findall(get_htmlx_mate_val,i)
                    if len(htmlx_mate_val)==1:
                        if htmlx_mate_name[0]!='':
                            #print(htmlx_mate_name[0])
                            #print(htmlx_mate_val[0])
                            #htmlx_mate_str+="\""+htmlx_mate_name[0]+"\":\""+htmlx_mate_val[0]+"\","
                            htmlx_mate_js[htmlx_mate_name[0]]=htmlx_mate_val[0]
                            htmlx_mate.append(htmlx_mate_val[0])
            htmlx_mate_str=json.dumps(htmlx_mate_js)

            #获取所有标签内容
            get_htmlx_all_tab='>([\s\S]*?)<'
            htmlx_tab=re.findall(get_htmlx_all_tab,htmlx)
            #print(htmlx_tab)

            #inpute mate
            for i in htmlx_mate:
                htmlx_tab.append(i)

            #先干掉\r\n\t 保留其他标点符号做语义识别
            get_text_rnt=['\r','\n','\t','  ']
            htmlx_tab_len=len(htmlx_tab)
            for i in range(htmlx_tab_len-1,-1,-1):#从后往前数
                for t in get_text_rnt:htmlx_tab[i]=htmlx_tab[i].replace(t,"")
                htmlx_tab[i]=htmlx_tab[i].strip()
                if htmlx_tab[i]=='':del htmlx_tab[i]#删除空
            #print(htmlx_tab)

            get_htmlx_BDFH=["\"","\\","^","’","=","/","、","“","”","#","©","|","_","-"," ","*",";","&","$","%","!","?",",",".","(",")","[","]","{","}","<",">","￥","%","！","？","，","。","（","）","【","】","《","》","："]
            #提取关键词
            htmlx_sents_js={}
            htmlx_words_js={}
            sents=[]
            words=[]
            words_int=0
            links_js={}
            links_int=0
            sents_int=0
            for i in range(len(htmlx_tab)):
                sent=htmlx_tab[i]              #获取每个句子
                word=jieba.cut_for_search(sent)#从每个句子获取单词
                #数据库限制每个句子256字符
                sent=sent[0:256]
                if det_repeat(sents,sent)==-1: #查重
                    sents.append(sent)
                    htmlx_sents_js[str(sents_int)]=sent
                    
                    for t in word:
                        #删除一些意义不大的符号
                        #数据库限制每个单词8字符
                        w=t[0:8]
                        for n in get_htmlx_BDFH:
                            w=w.replace(n,"")
                        if w!='':
                            words_det_int=-1
                            words_det_int=det_repeat(words,w)
                            if words_det_int==-1:
                                words.append(w)                       
                                words_det_int=words_int
                                htmlx_words_js[str(words_int)]=w
                                words_int+=1
                            links_js[str(links_int)]=str(words_det_int)+":"+str(sents_int)
                            links_int+=1
                    sents_int+=1

            htmlx_sents_str=json.dumps(htmlx_sents_js)
            htmlx_words_str=json.dumps(htmlx_words_js)
            links_str=json.dumps(links_js)


            
            if len(htmlx_title)==0:
                retunrn_js['title']=begin_url
            else:
                retunrn_js['title']=htmlx_title[0]   
            retunrn_js['mate']=htmlx_mate_str
            retunrn_js['sents']=htmlx_sents_str
            retunrn_js['words']=htmlx_words_str
            retunrn_js['links']=links_str
            retunrn_js['images']=image_url_str
            retunrn_js['new_url']=new_url_str
            retunrn_js['htmlx']=htmlx_kuaizhao

            print("分离数据成功")
            print("url:")
            print(retunrn_js['url'])
            print("status code:")
            print(retunrn_js['status'])
            print("content type:")
            print(retunrn_js['contenttype'])
            print("Title:")
            print(retunrn_js['title'])
            print("Sents const:")
            print(sents_int)
            print("Words const:")
            print(words_int)
            print("Links const:")
            print(links_int)
        else:
            retunrn_js['title']=begin_url
            print("分离数据成功 no html")
            print("url:")
            print(retunrn_js['url'])
            print("status code:")
            print(retunrn_js['status'])
            print("content type:")
            print(retunrn_js['contenttype'])
            print("Title:")
            print(retunrn_js['title'])
        r.close()
    except:
        print("连接失败")
        retunrn_js['status']=0
        retunrn_js['contenttype']="Cant connect"


    return retunrn_js


def getandpost(bgurl):
    page_data={}
    page_data=get_page_words(bgurl)
    page_data['password']="(*&RV^*(&VRH*(V)))"
    print("本地爬取完成,开始发送\n")
    post_url="https://lmve.net/php/reptile.php"
    r1 = requests.post(post_url,data=page_data)
    if r1.status_code==200:       
        html_byt=r1.content
        html_text=str(html_byt,'utf-8')
        print("发送成功:200\n")
        print(html_text)
        r1.close()
    else:
        print("error"+str(r.status_code))
    
def getoneurl():
    page_data={}
    page_data['password']="(*&RV^*(&VRH*(V)))"
    post_url="https://lmve.net/php/getoneurl.php"
    try :
        r = requests.post(post_url,data=page_data)
        if r.status_code==200:       
            html_byt=r.content
            html_text=str(html_byt,'utf-8')
            print("获取url成功:")
            print(html_text)
            print("\n")
            r.close()
            return html_text
        else:
            print("获取url失败\n")
            return "-1"
    except:
        print("获取url失败\n")
        return "-1"




theurl="https://lmve.net"
redata=get_page_words(theurl)
print(redata['words'])