init

Signed-off-by: kevin <kevin@dsm.lmve.net>

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.vs

myfunsion.py

@@ -0,0 +1,417 @@
+import requests
+import re
+import html
+import json
+import jieba
+from urllib.parse import urlparse
+class myfunsion:
+    def a():
+        print("test")
+
+class det:#测试url类型
+    def image(url):
+        style_type_list=[".jpg",".jpeg",".png",".gif",".svg"]
+        #将url转小写
+        url=url.lower()
+        for i in style_type_list:
+            if i in url:
+                return True
+        return False
+
+    def style(url):
+        style_type_list=[".js",".css"]
+        url=url.lower()
+        for i in style_type_list:
+            if i in url:
+                return True
+        return False
+
+    #检查是否有重复
+    # arr 数组
+    # i   数据
+    def repeat(arr,i):
+        b=0
+        for a in arr:
+            if a==i:
+                return b
+            b+=1
+        return -1
+
+    #检测url域名
+    def domin(url):
+        domin=urlparse(url)
+        return domin
+
+class net:
+    headers = {
+    'user-agent': 'my-app/0.0.1',
+    'Content-Type': 'text/html; charset=utf-8'
+    }
+    def get_html(url):      
+        return_js={}
+        return_js['url']=url
+        try :
+            r = requests.get(url,headers=net.headers,verify=False,timeout=(60, 60))
+            return_js['status']=r.status_code
+            get_content_type='text/html'
+            content_type=re.findall(get_content_type,r.headers['Content-Type'])
+            if(len(content_type)==0):     
+                return_js['contenttype']=r.headers['Content-Type']
+            else:
+                return_js['contenttype']=get_content_type
+            
+            if(return_js['contenttype']=='text/html'):
+                get_gb2312='gb2312'
+                charset=re.findall(get_gb2312,r.headers['Content-Type'])
+                if(len(charset)==0):     
+                    return_js['charset_type']='utf-8'
+                else:
+                    return_js['charset_type']='gb2312' 
+                htmlx_byt=r.content
+                htmlx=str(htmlx_byt,return_js['charset_type'])
+                return_js['html']=htmlx
+        except:
+            return_js['status']=0
+
+        return return_js
+
+    def ana_html(webdata):
+        return_js={}
+        return_js['status']=webdata['status']
+        return_js['url']=webdata['url']
+        if webdata['status']==0:
+            return_js['score']="0"
+            return_js['contenttype']="none"
+            return return_js
+        return_js['contenttype']=webdata['contenttype']
+        #return_js['charset_type']=webdata['charset_type']
+        if(return_js['contenttype']=='text/html'):
+            thisurlscore=100
+            htmlx=webdata['html']    
+            ur=urlparse(return_js['url'])
+            domain=ur.netloc
+            return_js['domain']=domain
+
+            
+            #根url
+            if(ur.path==''):
+                thisurlscore+=50
+            if(ur.path=='/'):
+                thisurlscore+=50
+            #是否带参数
+            if(ur.query!=''):
+                thisurlscore-=10
+            #是否带位置
+            if(ur.fragment!=''):
+                thisurlscore-=50
+            #获取js
+            get_htmlx_js='<script[\s\S]*?</script>'
+            htmlx_js=re.findall(get_htmlx_js,htmlx)
+            #清除htmlx js
+            for i in htmlx_js:
+                htmlx=htmlx.replace(i,"")
+
+            htmlx_kuaizhao=html.escape(htmlx) #干掉js后截取快照
+            htmlx_yasuo=["\n","\r","\t","  "]
+            for i in htmlx_yasuo:
+                htmlx_kuaizhao=htmlx_kuaizhao.replace(i,"")#压缩一下
+
+            #获取style
+            get_htmlx_style='<style[\s\S]*?</style>'
+            htmlx_style=re.findall(get_htmlx_style,htmlx)
+            #print(htmlx_style)
+            #清除htmlx style
+            for i in htmlx_style:
+                htmlx=htmlx.replace(i,"")
+
+            #获取textarea ,像百度这样的阴间网站有隐藏的输入框里面会有样式什么的奇怪东西
+            get_htmlx_textarea='<textarea[\s\S]*?</textarea>'
+            htmlx_textarea=re.findall(get_htmlx_textarea,htmlx)
+            for i in htmlx_textarea:
+                htmlx=htmlx.replace(i,"")
+            
+            #获取页面中的url并分类
+            get_url=r'\"(http[s]?://[\S]*)\"'
+            all_url=re.findall(get_url,htmlx)
+            style_url_list=[]
+            image_url_list=[]
+            new_url_list=[]
+            for i in all_url:
+                if det.style(i):
+                    style_url_list.append(i)
+                else:
+                    if det.image(i):
+                        image_url_list.append(i)
+                    else:
+                        new_url_list.append(i)
+            
+            #干掉url
+            image_url=[]
+            new_url=[]
+            image_url_js={}
+            new_url_js={}
+            image_url_int=0
+            new_url_int=0
+            for i in style_url_list:
+                htmlx=htmlx.replace(i,"")
+            for i in image_url_list:
+                htmlx=htmlx.replace(i,"")
+                if det.repeat(image_url,i)==-1:
+                    image_url.append(i)
+                    image_url_js[str(image_url_int)]=i
+                    image_url_int+=1
+                    thisurlscore+=2#图片加分
+            for i in new_url_list:
+                htmlx=htmlx.replace(i,"")
+                if det.repeat(new_url,i)==-1:
+                    new_url.append(i)
+                    new_url_js[str(new_url_int)]=i
+                    new_url_int+=1
+
+            #pending
+            image_url_str=json.dumps(image_url_js)
+            new_url_str=json.dumps(new_url_js)
+            return_js['images']=image_url_str
+            return_js['newurls']=new_url_str
+
+            #对新url评分
+            new_urlscore_js={}
+            new_urldomain_js={}
+            new_url_int=0
+            for i in new_url:
+                score=100
+                try :
+                    te=urlparse(i)
+
+                    #根url
+                    if(te.path==''):
+                        score+=10
+                    if(te.path=='/'):
+                        score+=10
+
+                    #是否站内url
+                    if(te.netloc==domain):
+                        score-=10
+                    else:
+                        score+=10
+                        thisurlscore+=5#站外url加分
+
+                    #是否带参数
+                    if(te.query!=''):
+                        score-=10
+
+                    #是否带位置
+                    if(te.fragment!=''):
+                        score-=50
+                except:
+                    score=0
+
+                new_urlscore_js[str(new_url_int)]=score
+                new_urldomain_js[str(new_url_int)]=te.netloc
+                new_url_int+=1
+            
+            new_urlscore_js_str=json.dumps(new_urlscore_js)
+            new_urldomain_js_str=json.dumps(new_urldomain_js)
+            return_js['newurlscore']=new_urlscore_js_str
+            return_js['newurldomain']=new_urldomain_js_str
+
+
+
+            #获取标题
+            get_htmlx_title='<title[\s\S]*?>([\s\S]*?)</title>'
+            htmlx_title=re.findall(get_htmlx_title,htmlx)
+            if(len(htmlx_title)>0):
+                return_js['title']=htmlx_title[0]
+                print(htmlx_title)
+            else:
+                return_js['title']=return_js['url']
+
+            #get all mate
+            get_htmlx_mate_lab='<meta[\S ]*?>'
+            htmlx_mate_lab=re.findall(get_htmlx_mate_lab,htmlx)
+            htmlx_mate=[]
+            htmlx_mate_str=""
+            htmlx_mate_js={}
+            for i in htmlx_mate_lab:
+                get_htmlx_mate_name='name="([\S ]*?)"'
+                htmlx_mate_name=re.findall(get_htmlx_mate_name,i)
+                if len(htmlx_mate_name)==1:#只允许有1个名字 其他阴间标签不要
+                    get_htmlx_mate_val='content="([\S\s]*?)"'
+                    htmlx_mate_val=re.findall(get_htmlx_mate_val,i)
+                    if len(htmlx_mate_val)==1:
+                        if htmlx_mate_name[0]!='':
+                            #print(htmlx_mate_name[0])
+                            #print(htmlx_mate_val[0])
+                            #htmlx_mate_str+="\""+htmlx_mate_name[0]+"\":\""+htmlx_mate_val[0]+"\","
+                            htmlx_mate_js[htmlx_mate_name[0]]=htmlx_mate_val[0]
+                            htmlx_mate.append(htmlx_mate_val[0])
+            htmlx_mate_str=json.dumps(htmlx_mate_js)
+            return_js['mate']=htmlx_mate_str
+
+            #获取所有标签内容
+            get_htmlx_all_tab='>([\s\S]*?)<'
+            htmlx_tab=re.findall(get_htmlx_all_tab,htmlx)
+            #print(htmlx_tab)
+
+            #inpute mate
+            for i in htmlx_mate:
+                htmlx_tab.append(i)
+
+            #先干掉\r\n\t 保留其他标点符号做语义识别
+            get_text_rnt=['\r','\n','\t','  ']
+            htmlx_tab_len=len(htmlx_tab)
+            for i in range(htmlx_tab_len-1,-1,-1):#从后往前数
+                for t in get_text_rnt:htmlx_tab[i]=htmlx_tab[i].replace(t,"")
+                htmlx_tab[i]=htmlx_tab[i].strip()
+                if htmlx_tab[i]=='':del htmlx_tab[i]#删除空
+            #print(htmlx_tab)
+
+            get_htmlx_BDFH=["\"","\\","^","’","=","/","、","“","”","#","©","|","_","-"," ","*",";","&","$","%","!","?",",",".","(",")","[","]","{","}","<",">","￥","%","！","？","，","。","（","）","【","】","《","》","："]
+            #提取关键词
+            htmlx_sents_js={}
+            htmlx_words_js={}
+            sents=[]
+            words=[]
+            words_int=0
+            links_js={}
+            links_int=0
+            sents_int=0
+            for i in range(len(htmlx_tab)):
+                sent=htmlx_tab[i]              #获取每个句子
+                word=jieba.cut_for_search(sent)#从每个句子获取单词
+                #数据库限制每个句子256字符
+                sent=sent[0:256]
+                if det.repeat(sents,sent)==-1: #查重
+                    sents.append(sent)
+                    htmlx_sents_js[str(sents_int)]=sent
+                    
+                    for t in word:
+                        #删除一些意义不大的符号
+                        #数据库限制每个单词8字符
+                        w=t[0:8]
+                        for n in get_htmlx_BDFH:
+                            w=w.replace(n,"")
+                        if w!='':
+                            words_det_int=-1
+                            words_det_int=det.repeat(words,w)
+                            if words_det_int==-1:
+                                words.append(w)                       
+                                words_det_int=words_int
+                                htmlx_words_js[str(words_int)]=w
+                                words_int+=1
+                            links_js[str(links_int)]=str(words_det_int)+":"+str(sents_int)
+                            links_int+=1
+                    sents_int+=1
+                else:
+                    #重复的内容
+                    thisurlscore-=1
+
+            return_js['sents_int']=sents_int
+            return_js['words_int']=words_int
+            return_js['links_int']=links_int            
+
+            htmlx_sents_str=json.dumps(htmlx_sents_js)
+            htmlx_words_str=json.dumps(htmlx_words_js)
+            links_str=json.dumps(links_js)
+
+            return_js['sents']=htmlx_sents_str
+            return_js['words']=htmlx_words_str
+            return_js['links']=links_str
+
+            return_js['htmlx']=htmlx_kuaizhao
+            return_js['score']=thisurlscore#对当前url评分
+        else:
+            return_js['title']=return_js['url']
+        return return_js
+
+
+
+
+
+
+
+
+test=0
+
+getone={}
+getone['password']="(*&RV^*(&VRH*(V)))"
+getone_post_url="https://lmve.net/php/getoneurl.php"
+post_url="https://lmve.net/php/newurlreptile.php"
+
+if test==0:
+    while 1:
+        print("**本地消息**\n获取url..")
+        try:
+            r = requests.post(getone_post_url,data=getone)
+            if r.status_code==200:       
+                html_byt=r.content
+                html_text=str(html_byt,'utf-8')
+                print("获取url成功:")
+                print(html_text)
+                print("\n")
+                r.close()
+                if html_text!='':
+                    print("获取text..")
+                    text=net.get_html(html_text)
+                    print("开始分析..")
+                    postdata=net.ana_html(text)                   
+                    print("分析完成")
+                    postdata['password']="(*&RV^*(&VRH*(V)))"
+                    if postdata['contenttype']=='text/html':
+                        print("分离数据成功")
+                        print("url:")
+                        print(postdata['url'])
+                        print("status code:")
+                        print(postdata['status'])
+                        print("content type:")
+                        print(postdata['contenttype'])
+                        print("Title:")
+                        print(postdata['title'])
+                        print("Sents const:")
+                        print(postdata['sents_int'])
+                        print("Words const:")
+                        print(postdata['words_int'])
+                        print("Links const:")
+                        print(postdata['links_int'])
+                    else :
+                        print("分离数据成功 no html")
+                        print("url:")
+                        print(postdata['url'])
+                        print("status code:")
+                        print(postdata['status'])
+                        print("content type:")
+                        print(postdata['contenttype'])
+                    print("开始发送..")
+                    try:
+                        r1 = requests.post(post_url,data=postdata)
+                        if r1.status_code==200:       
+                            html_byt=r1.content
+                            html_text=str(html_byt,'utf-8')
+                            print("发送成功:200")
+                            print(html_text)
+                            r1.close()
+                        else:
+                            print("error"+str(r1.status_code))            
+                    except:
+                        print("try aga")
+            else:
+                print("获取url失败")
+        except:
+            print("tyr agin")
+            
+       
+
+
+
+if test==1:
+    postdata=net.ana_html(net.get_html("https://www.foreverblog.cn/blog/1958.html"))
+    postdata['password']="(*&RV^*(&VRH*(V)))"   
+    r1 = requests.post(post_url,data=postdata)
+    if r1.status_code==200:       
+        html_byt=r1.content
+        html_text=str(html_byt,'utf-8')
+        print("发送成功:200\n")
+        print(html_text)
+        r1.close()
+    else:
+        print("error"+str(r1.status_code))

post.py

@@ -0,0 +1,19 @@
+# encoding:utf-8
+import requests
+def main():
+    post_url="https://lmve.net/php/reptile.php"
+    headers = {
+        'user-agent': 'my-app/0.0.1',
+        'Content-Type': 'text/html; charset=utf-8'
+        }
+    data={}
+    data['password']="adswewd"
+    
+    r = requests.post(post_url,data=data)
+    if r.status_code!=200:
+        return r.status_code
+    html_byt=r.content
+    html=str(html_byt,'utf-8')
+    print(html)
+
+main()

reptile.py

@@ -0,0 +1,374 @@
+# encoding:utf-8
+#简易爬虫
+
+#     requests  用于下载网页源码  第三方库 通过pip安装  
+#     re        正则表达式要用到
+#     threading 启用多线程
+#     os，sys   用于创建目录
+#     time      用于获得当天时间
+from asyncio.windows_events import NULL
+import requests
+import re
+import jieba
+import html
+import json
+from myfunsion import myfunsion
+#from bs4 import BeautifulSoup as bs
+#from lxml import etree
+
+def det_image(url):
+    style_type_list=[".jpg",".jpeg",".png",".gif"]
+    #将url转小写
+    url=url.lower()
+    for i in style_type_list:
+        if i in url:
+            return True
+    return False
+
+def det_style(url):
+    style_type_list=[".js",".css"]
+    url=url.lower()
+    for i in style_type_list:
+        if i in url:
+            return True
+    return False
+
+def det_repeat(arr,i):
+    b=0
+    for a in arr:
+        if a==i:
+            return b
+        b+=1
+    return -1
+
+def get_page_words(url):
+
+    retunrn_js={}
+    begin_url=url
+    headers = {
+        'user-agent': 'my-app/0.0.1',
+        'Content-Type': 'text/html; charset=utf-8'
+        }
+
+    retunrn_js['url']=begin_url
+    #print(begin_url)
+    #获取整个页面
+    try:
+        r = requests.get(begin_url,headers=headers,verify=False)
+        retunrn_js['status']=r.status_code
+        get_content_type='text/html'
+        content_type=re.findall(get_content_type,r.headers['Content-Type'])
+        if(len(content_type)==0):     
+            retunrn_js['contenttype']=r.headers['Content-Type']
+        else:
+            retunrn_js['contenttype']=get_content_type
+
+        get_gb2312='gb2312'
+        charset=re.findall(get_gb2312,r.headers['Content-Type'])
+        if(len(charset)==0):     
+            charset_type='utf-8'
+        else:
+            charset_type='gb2312'    
+
+        if(retunrn_js['contenttype']=='text/html'):
+
+            htmlx_byt=r.content
+            htmlx=str(htmlx_byt,charset_type)
+            #print(htmlx)
+
+            #获取js
+            get_htmlx_js='<script[\s\S]*?</script>'
+            htmlx_js=re.findall(get_htmlx_js,htmlx)
+            #清除htmlx js
+            for i in htmlx_js:
+                htmlx=htmlx.replace(i,"")
+
+            htmlx_kuaizhao=html.escape(htmlx) #干掉js后截取快照
+            htmlx_yasuo=["\n","\r","\t","  "]
+            for i in htmlx_yasuo:
+                htmlx_kuaizhao=htmlx_kuaizhao.replace(i,"")#压缩一下
+
+            #获取style
+            get_htmlx_style='<style[\s\S]*?</style>'
+            htmlx_style=re.findall(get_htmlx_style,htmlx)
+            #print(htmlx_style)
+            #清除htmlx style
+            for i in htmlx_style:
+                htmlx=htmlx.replace(i,"")
+
+            #print(htmlx)
+
+            #获取textarea ,像百度这样的阴间网站有隐藏的输入框里面会有样式什么的奇怪东西
+            get_htmlx_textarea='<textarea[\s\S]*?</textarea>'
+            htmlx_textarea=re.findall(get_htmlx_textarea,htmlx)
+            for i in htmlx_textarea:
+                htmlx=htmlx.replace(i,"")
+
+            #获取页面中的url并分类
+            get_url=r'"(http[s]?://[\S]*)"'
+            all_url=re.findall(get_url,htmlx)
+            style_url_list=[]
+            image_url_list=[]
+            new_url_list=[]
+            for i in all_url:
+                if det_style(i):
+                    style_url_list.append(i)
+                else:
+                    if det_image(i):
+                        image_url_list.append(i)
+                    else:
+                        new_url_list.append(i)
+
+            #干掉url
+            image_url=[]
+            new_url=[]
+            image_url_js={}
+            new_url_js={}
+            image_url_int=0
+            new_url_int=0
+            for i in style_url_list:
+                htmlx=htmlx.replace(i,"")
+            for i in image_url_list:
+                htmlx=htmlx.replace(i,"")
+                if det_repeat(image_url,i)==-1:
+                    image_url.append(i)
+                    image_url_js[str(image_url_int)]=i
+                    image_url_int+=1
+            for i in new_url_list:
+                htmlx=htmlx.replace(i,"")
+                if det_repeat(new_url,i)==-1:
+                    new_url.append(i)
+                    new_url_js[str(new_url_int)]=i
+                    new_url_int+=1
+
+            image_url_str=json.dumps(image_url_js)
+            new_url_str=json.dumps(new_url_js)
+
+            #获取标签内容
+            #soup=bs(htmlx,'htmlx.parser') 
+            #dom = etree.htmlx(htmlx)
+            #print(soup.body)
+
+            #获取标题
+            get_htmlx_title='<title[\s\S]*?>([\s\S]*?)</title>'
+            htmlx_title=re.findall(get_htmlx_title,htmlx)
+            #print(htmlx_title)
+
+            #get all mate
+            get_htmlx_mate_lab='<meta[\S ]*?>'
+            htmlx_mate_lab=re.findall(get_htmlx_mate_lab,htmlx)
+            htmlx_mate=[]
+            htmlx_mate_str=""
+            htmlx_mate_js={}
+            for i in htmlx_mate_lab:
+                get_htmlx_mate_name='name="([\S ]*?)"'
+                htmlx_mate_name=re.findall(get_htmlx_mate_name,i)
+                if len(htmlx_mate_name)==1:#只允许有1个名字 其他阴间标签不要
+                    get_htmlx_mate_val='content="([\S\s]*?)"'
+                    htmlx_mate_val=re.findall(get_htmlx_mate_val,i)
+                    if len(htmlx_mate_val)==1:
+                        if htmlx_mate_name[0]!='':
+                            #print(htmlx_mate_name[0])
+                            #print(htmlx_mate_val[0])
+                            #htmlx_mate_str+="\""+htmlx_mate_name[0]+"\":\""+htmlx_mate_val[0]+"\","
+                            htmlx_mate_js[htmlx_mate_name[0]]=htmlx_mate_val[0]
+                            htmlx_mate.append(htmlx_mate_val[0])
+            htmlx_mate_str=json.dumps(htmlx_mate_js)
+
+            #获取所有标签内容
+            get_htmlx_all_tab='>([\s\S]*?)<'
+            htmlx_tab=re.findall(get_htmlx_all_tab,htmlx)
+            #print(htmlx_tab)
+
+            #inpute mate
+            for i in htmlx_mate:
+                htmlx_tab.append(i)
+
+            #先干掉\r\n\t 保留其他标点符号做语义识别
+            get_text_rnt=['\r','\n','\t','  ']
+            htmlx_tab_len=len(htmlx_tab)
+            for i in range(htmlx_tab_len-1,-1,-1):#从后往前数
+                for t in get_text_rnt:htmlx_tab[i]=htmlx_tab[i].replace(t,"")
+                htmlx_tab[i]=htmlx_tab[i].strip()
+                if htmlx_tab[i]=='':del htmlx_tab[i]#删除空
+            #print(htmlx_tab)
+
+            get_htmlx_BDFH=["\"","\\","^","’","=","/","、","“","”","#","©","|","_","-"," ","*",";","&","$","%","!","?",",",".","(",")","[","]","{","}","<",">","￥","%","！","？","，","。","（","）","【","】","《","》","："]
+            #提取关键词
+            htmlx_sents_js={}
+            htmlx_words_js={}
+            sents=[]
+            words=[]
+            words_int=0
+            links_js={}
+            links_int=0
+            sents_int=0
+            for i in range(len(htmlx_tab)):
+                sent=htmlx_tab[i]              #获取每个句子
+                word=jieba.cut_for_search(sent)#从每个句子获取单词
+                #数据库限制每个句子256字符
+                sent=sent[0:256]
+                if det_repeat(sents,sent)==-1: #查重
+                    sents.append(sent)
+                    htmlx_sents_js[str(sents_int)]=sent
+                    
+                    for t in word:
+                        #删除一些意义不大的符号
+                        #数据库限制每个单词8字符
+                        w=t[0:8]
+                        for n in get_htmlx_BDFH:
+                            w=w.replace(n,"")
+                        if w!='':
+                            words_det_int=-1
+                            words_det_int=det_repeat(words,w)
+                            if words_det_int==-1:
+                                words.append(w)                       
+                                words_det_int=words_int
+                                htmlx_words_js[str(words_int)]=w
+                                words_int+=1
+                            links_js[str(links_int)]=str(words_det_int)+":"+str(sents_int)
+                            links_int+=1
+                    sents_int+=1
+
+            htmlx_sents_str=json.dumps(htmlx_sents_js)
+            htmlx_words_str=json.dumps(htmlx_words_js)
+            links_str=json.dumps(links_js)
+
+
+            
+            if len(htmlx_title)==0:
+                retunrn_js['title']=begin_url
+            else:
+                retunrn_js['title']=htmlx_title[0]   
+            retunrn_js['mate']=htmlx_mate_str
+            retunrn_js['sents']=htmlx_sents_str
+            retunrn_js['words']=htmlx_words_str
+            retunrn_js['links']=links_str
+            retunrn_js['images']=image_url_str
+            retunrn_js['new_url']=new_url_str
+            retunrn_js['htmlx']=htmlx_kuaizhao
+
+            print("分离数据成功")
+            print("url:")
+            print(retunrn_js['url'])
+            print("status code:")
+            print(retunrn_js['status'])
+            print("content type:")
+            print(retunrn_js['contenttype'])
+            print("Title:")
+            print(retunrn_js['title'])
+            print("Sents const:")
+            print(sents_int)
+            print("Words const:")
+            print(words_int)
+            print("Links const:")
+            print(links_int)
+        else:
+            retunrn_js['title']=begin_url
+            print("分离数据成功 no html")
+            print("url:")
+            print(retunrn_js['url'])
+            print("status code:")
+            print(retunrn_js['status'])
+            print("content type:")
+            print(retunrn_js['contenttype'])
+            print("Title:")
+            print(retunrn_js['title'])
+        r.close()
+    except:
+        print("连接失败")
+        retunrn_js['status']=0
+        retunrn_js['contenttype']="Cant connect"
+
+
+    return retunrn_js
+
+
+def getandpost(bgurl):
+    page_data={}
+    page_data=get_page_words(bgurl)
+    page_data['password']="(*&RV^*(&VRH*(V)))"
+    print("本地爬取完成,开始发送\n")
+    post_url="https://lmve.net/php/reptile.php"
+    r1 = requests.post(post_url,data=page_data)
+    if r1.status_code==200:       
+        html_byt=r1.content
+        html_text=str(html_byt,'utf-8')
+        print("发送成功:200\n")
+        print(html_text)
+        r1.close()
+    else:
+        print("error"+str(r.status_code))
+    
+def getoneurl():
+    page_data={}
+    page_data['password']="(*&RV^*(&VRH*(V)))"
+    post_url="https://lmve.net/php/getoneurl.php"
+    try :
+        r = requests.post(post_url,data=page_data)
+        if r.status_code==200:       
+            html_byt=r.content
+            html_text=str(html_byt,'utf-8')
+            print("获取url成功:")
+            print(html_text)
+            print("\n")
+            r.close()
+            return html_text
+        else:
+            print("获取url失败\n")
+            return "-1"
+    except:
+        print("获取url失败\n")
+        return "-1"
+
+
+
+test=1
+
+if test==0:
+    while 1:
+        theurl=getoneurl()
+        if theurl!='-1':
+            getandpost(theurl)
+        else :
+            print("获取url失败,自动重试")
+
+if test==1:
+    theurl="https://lmve.net"
+    getandpost(theurl)
+
+if test==2:
+    theurl=getoneurl()
+    print(theurl)
+    getandpost(theurl)
+
+if test==3:
+    headers = {
+    'user-agent': 'my-app/0.0.1',
+    'Content-Type': 'text/html; charset=utf-8'
+    }
+
+    #print(begin_url)
+    #获取整个页面
+    url1="https://git.lmve.net/kevin/um-all-index-web/-/avatar"
+    url2="https://yyyyyyounger.com/"
+    try :
+        r = requests.get(url2,headers=headers,verify=False)
+        htmlx_byt=r.content
+        print(r.headers['Content-Type'])
+    except:
+        print("11")
+if test==4:
+    #js=get_page_words("https://lmve.net")
+    #print(js)
+    myfunsion.a()
+    
+    
+    
+
+    
+
+
+    
+
+

搜索引擎工作思路.md

@@ -0,0 +1,29 @@
+# 工作流程
+
+## 数据库存储结构
+
+  url->标签->单词
+
+## 爬虫端(新url爬虫)
+1. 从数据库的将要爬取的ulr表获取一条url
+2. 获取url的html,分析连接'标签'与'单词'
+3. 获取url domin,分析新url与站内url
+4. 对新url打分,如果是站内url就扣分,(抑制爬虫在同一站点的爬取欲望)
+5. 将数据发送到后端
+
+## 后端(接收新url爬虫)
+1. 接收数据,分离'标签'与'单词'
+2. 首先查询'标签'是否在数据库中存在,如果存在说明'标签'对应的'单词'也是存在的,这样就不必再往数据库写入内容,同时说明有重复的内容,要扣内容分
+3. 如果标签不存在,写入标签,写入对应的单词,单词有可能是重复的,所以要在本地建立一个表记录单词在数据库中的id,链接标签与单词时优先使用本地表
+4. 统计标签重复率,按百分比给内容打分,比如重复率是%100就是0分,重复率0%就是100分,分数越低,搜索排名越靠后.
+
+## 搜索数据
+1. 将搜索内容分离成单词通过 单词->标签->url 的方式获取最终url并按照内容分排序
+2. 如果url被点击,内容分加1
+
+## 爬虫端(优质内容更新)
+1. 时间倒序内容分正序,获取一条url爬取
+2. 将数据发送到后端
+
+## 后端(接收优质内容更新)
+1. 对内容进行评分,重复的标签不扣分,如果标签相比之前少了就扣分,多了就加分