网页视频抓取脚本(用python实现的抓取腾讯视频所有电影的爬虫总结以上)

　　网页视频抓取脚本(用python实现的抓取腾讯视频所有电影的爬虫总结以上)

　　python实现的一个爬取所有腾讯视频电影的爬虫

<p>

# -*- coding: utf-8 -*-

import re

import urllib2

from bs4import BeautifulSoup

import string, time

import pymongo

NUM =0 #全局变量,电影数量

m_type = u'' #全局变量,电影类型

m_site = u'qq' #全局变量,电影网站

#根据指定的URL获取网页内容

def gethtml(url):

req = urllib2.Request(url)

response = urllib2.urlopen(req)

html = response.read()

return html

#从电影分类列表页面获取电影分类

def gettags(html):

global m_type

soup = BeautifulSoup(html) #过滤出分类内容

#print soup

#

tags_all = soup.find_all('ul', {'class' :'clearfix _group' ,'gname' :'mi_type'})

#print len(tags_all), tags_all

#print str(tags_all[1]).replace('\n','')

#动作

re_tags = r'<a _hot=\"tag\.sub\" class=\"_gtag _hotkey\" href=\"(.+?)\" title=\"(.+?)\" tvalue=\"(.+?)\">.+?</a>'

p = re.compile(re_tags, re.DOTALL)

tags = p.findall(str(tags_all[0]))

if tags:

tags_url = {}

#print tags

for tagin tags:

tag_url = tag[0].decode('utf-8')

#print tag_url

m_type = tag[1].decode('utf-8')

tags_url[m_type] = tag_url

else:

print"Not Find"

return tags_url

#获取每个分类的页数

def get_pages(tag_url):

tag_html = gethtml(tag_url)

#divclass="paginator

soup = BeautifulSoup(tag_html) #过滤出标记页面的html

#print soup

#

div_page = soup.find_all('div', {'class' :'mod_pagenav','id' :'pager'})

#print div_page #len(div_page), div_page[0]

#25</a>

re_pages = r'<a class=.+?>(.+?)</a>'

p = re.compile(re_pages, re.DOTALL)

pages = p.findall(str(div_page[0]))

#print pages

if len(pages) >1:

return pages[-2]

else:

return 1

def getmovielist(html):

soup = BeautifulSoup(html)

#

divs = soup.find_all('ul', {'class' :'mod_list_pic_130'})

#print divs

for div_htmlin divs:

div_html = str(div_html).replace('\n','')

#print div_html

getmovie(div_html)

def getmovie(html):

global NUM

global m_type

global m_site

re_movie = r'<a class=\"mod_poster_130\" href=\"(.+?)\" target=\"_blank\" title=\"(.+?)\">