*获取单个网页评论发布时间、评论标题和urlclear allcap mkdir d:/豆瓣影评cd d:/豆瓣影评copy "https://movie.douban.com/subject/35196097/reviews?start=0" temp.txt,replace infix strL v 1-100000 using temp.txt, clear  preserve keep if index(v,`"<span content=""') //保留包含<span content="的行replace v=ustrregexra(v,"<.*?>","") //把<>之间的内容替换为空rename v time save time.dta,replace restore  keep if index(v,`"<h2><a href="') //保留包含<h2><a href="的行split v,p(`"""') //用双引号作为分隔符分割rename v2 url //重命名v2为urlsplit v3,p(">") //用>作为分隔符分割split v32,p("<") //用<作为分隔符分割rename v321 title keep url title  merge using time drop _mergesave "影评1.dta",replace 

*获取单个网页的评论内容clearcap mkdir d:/豆瓣影评/二次爬取 cd d:/豆瓣影评/二次爬取 use /Users/cc/Desktop/cc的推文/豆瓣影评/影评1.dta,clear gen v=""  forvalues i=1/`=_N'{    replace v=fileread(url) in `i'     while filereaderror(v[`i'])!=0 {       sleep 5000       replace v=fileread(url) in `i'    }    }split v,p(`"<div id="link-report">"' `"<div class="copyright">"') keep v2replace v2 = ustrregexra(v2,"\s","",.) //删去v2中的空白符replace v2 = ustrregexra(v2, "<.*?>", "",.) //把<>之间的内容替换为空rename v2 content save "影评内容1.dta",replace

clear allcap mkdir d:/豆瓣影评cd d:/豆瓣影评infix strL v 1-100000 using temp.txt, clear keep if index(v,`"<span class="thispage" data-total-page="')replace v=ustrregexs(0) if ustrregexm(v,"(\d+)")  local p=v[1]forvalues i=1/`p'{  local j=(`i'-1)*20   cap copy "https://movie.douban.com/subject/35196097/reviews?start=`j'" temp.txt,replace  infix strL v 1-100000 using temp.txt,clear  preserve   keep if index(v,`"<span content=""')  replace v=ustrregexra(v,"<.*?>","")  rename v time  save time.dta,replace  restore  keep if index(v,`"<h2><a href="')   split v,p(`"""')  rename v2 url  split v3,p(">")  split v32,p("<")  rename v321 title  keep url title  merge using time  drop _merge  save "影评`i'.dta",replace //第i条评论对应的url、title、time保存为影评i.dta  }  clearlocal files:dir "." file "影评*.dta"foreach file in `files'{  append using "`file'" //把上面生成的影评i.dta合并起来    }save "影评.dta",replace gen v=""  forvalues i=1/`=_N'{    replace v=fileread(url) in `i'    while filereaderror(v[`i'])!=0 {      sleep `=int(runiform()*5000+5000)' //随机休息5-10秒      replace v=fileread(url) in `i'    }  sleep `=int(runiform()*5000+5000)'    }split v,p(`"<div id="link-report">"' `"<div class="copyright">"')keep v2replace v2 = ustrregexra(v2,"\s","",.)replace v2 = ustrregexra(v2, "<.*?>", "",.)rename v2 contentmerge using "影评.dta"drop _mergeorder  (time url title content)save "影评.dta",replace

*分词处理*导出为txt文档use 影评.dta,clearkeep contentexport delimited using content.txt,replace  *调用Python分词clear allpythonimport jieba.possegword=[]with open(r"content.txt",encoding="utf-8") as f:  for i in f.readlines():    str=i    word.append(str)jieba.load_userdict(r"dict.txt")with open("分词.txt","w",encoding="utf-8") as f2:  for unit in word:    seg_list = jieba.posseg.cut(unit)     for word in seg_list:      f2.write(word.word+" "+word.flag+"\n")end *分词结果导入Stata，并删除单字、缺失值、停用词import delimited using 分词.txt, clear encoding("utf-8")split v1,p(" ")drop v1rename (v11 v12) (keyword flag)drop if ustrlen(keyword) == 1 // 删除单字drop if keyword =="" //删除缺失值preserveimport delimited using 停用词表.txt, clear ///  encoding("utf-8") varname(nonames)outsheet using 停用词表.txt, replace nonames noquotelevelsof v1, local(keyword)restoreforeach word in `keyword' {  drop if keyword == "`word'" //删除停用词}*词频统计bysort keyword: gen frequency = _Ngsort -frequencyduplicates dropsave word,replace*在分词结果中只保留动词、名词和形容词并绘制词云图：use word,clearkeep if ustrregexm(flag, "^[anv]")  keep in 1/100wordcloud keyword frequency using 词云.html, replace  size(15 80) range(3840 2160)shellout 词云.html