python爬虫

import xlwt   #excel操作
import urllib.request,urllib.error  #制定URL,获取网络数据
import sqlite3  #数据库操作
import bs4   #网页解析
from bs4 import BeautifulSoup
import re   #正则表达式

Urllib

主要进行request操作，获得响应体response

get请求

import urllib.request
response = urllib.request.urlopen("http://www.baidu.com/")
print(response.read().decode("utf-8"))  #获取网页源码，并以utf-8解码

post请求

模拟真实用户登录时使用

data = bytes(urllib.parse.urlencode({
     "user": "root",
     "password": "123456"
 }), encoding="utf-8")
 response = urllib.request.urlopen("http://httpbin.org/post", data=data)
 print(response.read().decode("utf-8"))

超时请求

 try:
     response = urllib.request.urlopen("http://httpbin.org/get", timeout=1)
     print(response.read().decode("utf-8"))
     # 获取详细内容
     print(response.status)
     # 获取状态码
     print(response.getheader("server"))
     # 获取响应头
 except urllib.error.URLError as e:
     print("time out...")

封装response

url = "http://httpbin.org/post"
data = bytes(urllib.parse.urlencode({
    "hello": "world"
}), encoding='utf-8')
headers = {
    "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"
}
req = urllib.request.Request(url=url, data=data, headers=headers, method="POST")
response = urllib.request.urlopen(req)
print(response.read().decode("utf-8"))

BeautifulSoup

对html文件进行搜索，遍历，查找

基本使用

from bs4 import BeautifulSoup

file = open("./baidu.html", "rb")
html = file.read().decode('utf-8')
# print(html)
bs = BeautifulSoup(html, "html.parser")
print(bs.title.string)  #获取title的string
print(bs.a.string)      
print(bs.a.attrs)   #获取 a class 内容  {'class': ['mnav'], 'href': 'http://news.baidu.com', 'name': 'tj_trnews'}
print(type(bs))  #<class 'bs4.BeautifulSoup'>
print(type(bs.a.string))  #<class 'bs4.element.Comment'>
print(type(bs.title.string))  #<class 'bs4.element.NavigableString'>
print(type(bs.a))  #<class 'bs4.element.Tag'>

文档搜索

find_all()

from bs4 import BeautifulSoup

file = open("./baidu.html", "rb")
html = file.read().decode('utf-8')
bs = BeautifulSoup(html, "html.parser")

# 字符串选择
t_list = bs.find_all("a")

#正则表达式
import re
t_list = bs.find_all(re.compile("a"))

#函数
def name_is(tag):
    return tag.has_attr("name")

t_list = bs.find_all(name_is)

select()

css选择器

from bs4 import BeautifulSoup

file = open("./baidu.html", "rb")
html = file.read().decode('utf-8')
bs = BeautifulSoup(html, "html.parser")

t_list = bs.select("#u1")  # id选择器
t_list = bs.select(".mnav")  # 类选择器
t_list = bs.select("title")  # 标签选择器

正则表达式

基础

re模块

compile(pattern)：创建模式对象

search(pattern,string)：在字符串中寻找模式

match(pattern,string)：在字符串开始处匹配模式

split(pattern,string)：根据模式分割字符串,返回列表

findall(pattern,string)：列表形式返回匹配项

sub(pat,repl,string) ：用repl替换 pat匹配项

import re
# search(pattern,string)在字符串中寻找模式

a = re.search("AA", "ASDAAASD")
print(a)
# <re.Match object; span=(3, 5), match='AA'>

# findall(pattern,string)：列表形式返回匹配项

b = re.findall("[a-z]", "as213bn23o123nd12d")
print(b)
# ['a', 's', 'b', 'n', 'o', 'n', 'd', 'd']

# sub(pat,repl,string) ：用repl替换 pat匹配项

c = re.sub("a", "A", "a12abi1ans")
print(c)
# A12Abi1Ans

Xlwt(Excel)

Python编写excel表

import xlwt


workbook = xlwt.Workbook(encoding='utf-8')  # 创建workbook对象
worksheet = workbook.add_sheet('sheet1')  # 创建工作表
worksheet.write(行, 列, 内容)  # 写入数据
workbook.save("student.xls")  # 保存数据表

Sqllite

建表,插入,查询

import sqlite3

# 创建或打开数据库
conn = sqlite3.connect("text.db")

print("数据库创建/打开成功！！！")

# # 创建表
# c = conn.cursor()  # 获得游标
#
# sql = '''
#    create table company
#         (id int primary key not null,
#           name text not null,
#           age int not null,
#           address char(50),
#           salary real );
# '''
#
# c.execute(sql)  # 执行sql语句
# conn.commit()  # 提交数据库操作
# conn.close()  # 关闭数据库连接
#
# print("数据表创建成功！！！")

# # 插入
# c = conn.cursor()  # 获得游标
#
# sql = '''
#     insert into company (id,name,age,address,salary)
#        valueS (1,'张三',32,'山东',8000);
# '''
#
# c.execute(sql)  # 执行sql语句
# conn.commit()  # 提交数据库操作
# conn.close()  # 关闭数据库连接
#
# print("数据插入成功！！！")

# 查询
c = conn.cursor()  # 获得游标

sql = '''
     select * from company;
'''

cursor = c.execute(sql)  # 执行sql语句
for i in cursor:
    print(i)

conn.close()  # 关闭数据库连接

print("数据查询成功！！！")