#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from urllib import request
import json
from datetime import datetime, timedelta
import time
def get_data():
url = 'https://search.51job.com/list/070200,000000,0000,00,9,99,java%25E5%25BC%2580%25E5%258F%2591,2,1.html'
headers = {
'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36'
}
req = request.Request(url, headers=headers)
response = request.urlopen(req)
print('===============', response.getcode(), type(response))
if response.getcode() == 200:
# 字节码
data = response.read()
print(type(data))
# 字节码转字符串
data = data.decode('utf-8')
print(type(data))
# print(data)
with open('index.html', mode='w', encoding='utf-8') as f:
f.write(data)
if __name__ == '__main__':
get_data()收集linux数据
通过python收集linux主机的cpu、硬盘、内存等
root@ubuntu16:/server/scripts# cat collect_linux_info.py
#!/usr/bin/env python3
# -*- coding:utf-8 -*-
import subprocess
def collect():
filter_keys = ['Manufacturer', 'Serial Number', 'Product Name', 'UUID', 'Wake-up Type']
raw_data = {}
for key in filter_keys:
try:
res = subprocess.Popen("sudo dmidecode -t system|grep '%s'" % key, stdout=subprocess.PIPE, shell=True)
# result = res.stdout.read().decode()等于下面这两句
result_bytes = res.stdout.read()
result = result_bytes.decode()
# split分隔字符,变成列表
data_list = result.split(':')
if len(data_list) > 1:
raw_data[key] = data_list[1].strip()
else:
raw_data[key] = ''
except Exception as e:
raw_data[key] = ''
# 构造符合django的数据模型字段
data = {}
data['asset_type'] = 'server'
data['manufacturer'] = raw_data['Manufacturer']
data['sn'] = raw_data['Serial Number']
data['model'] = raw_data['Product Name']
data['uuid'] = raw_data['UUID']
data['wake_up_type'] = raw_data['Wake-up Type']
data.update(get_os_info())
data.update(get_cpu_info())
data.update(get_ram_info())
data.update(get_nic_info())
data.update(get_disk_info())
return data
def get_os_info():
"""
获取操作系统信息
：return:
"""
distributor = subprocess.Popen("lsb_release -a|grep 'Distributor ID'", stdout=subprocess.PIPE, shell=True)
distributor = distributor.stdout.read().decode().split(":")
release = subprocess.Popen("lsb_release -a|grep 'Description'", stdout=subprocess.PIPE, shell=True)
release = release.stdout.read().decode().split(":")
data_dic = {
"os_distribution": distributor[1].strip() if len(distributor) > 1 else "",
"os_release": release[1].strip() if len(release) > 1 else"",
"os_type": "Linux",
}
return data_dic
def get_cpu_info():
"""
获取cpu信息
:return
"""
raw_cmd = 'cat /proc/cpuinfo'
raw_data = {
'cpu_model': "%s |grep 'model name' |head -1 " % raw_cmd,
'cpu_count': "%s | grep 'processor'|wc -l" % raw_cmd,
'cpu_core_count': "%s | grep 'cpu cores' |awk -F : '{SUM +=$2} END {print SUM}'" % raw_cmd,
}
for key, cmd in raw_data.items():
try:
result = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
raw_data[key] = result.stdout.read().decode().strip()
except ValueError as e:
print(e)
raw_data[key] = ""
data = {
"cpu_count": raw_data["cpu_count"],
"cpu_core_count": raw_data["cpu_core_count"]
}
cpu_model = raw_data["cpu_model"].split(":")
return data
def get_ram_info():
"""
获取内存信息
：return：
"""
raw_data = subprocess.Popen("sudo dmidecode -t memory", stdout=subprocess.PIPE, shell=True)
raw_list = raw_data.stdout.read().decode().split("\n")
raw_ram_list = []
item_list = []
for line in raw_list:
if line.startswith("Memory Device"):
raw_ram_list.append(item_list)
item_list = []
else:
item_list.append(line.strip())
ram_list = []
for item in raw_ram_list:
item_ram_size = 0
ram_item_to_dic = {}
for i in item:
data = i.split(":")
if len(data) == 2:
key, v = data
if key == 'Size':
if v.strip() != "No Module Installed":
ram_item_to_dic['capacity'] = v.split()[0].strip()
item_ram_size = round(int(v.split()[0]))
else:
ram_item_to_dic['capacity'] = 0
if key == 'Type':
ram_item_to_dic['model'] = v.strip()
if key == 'Manufacturer':
ram_item_to_dic['manufacturer'] = v.strip()
if key == 'Serial Number':
ram_item_to_dic['sn'] = v.strip()
if key == 'Asset Tag':
ram_item_to_dic['asset_tag'] = v.strip()
if key == 'Locator':
ram_item_to_dic['slot'] = v.strip()
if item_ram_size == 0:
pass
else:
raw_list.append(ram_item_to_dic)
raw_total_size = subprocess.Popen("cat /proc/meminfo|grep MemTotal", stdout=subprocess.PIPE, shell=True)
raw_total_size = raw_total_size.stdout.read().decode().split(":")
ram_data = {'ram': ram_list}
if len(raw_total_size) == 2:
total_gb_size = int(raw_total_size[1].split()[0]) / 1024**2
ram_data['ram_size'] = total_gb_size
return ram_data
def get_nic_info():
"""
获取网卡信息
:return:
"""
raw_data = subprocess.Popen("ifconfig -a", stdout=subprocess.PIPE, shell=True)
raw_data = raw_data.stdout.read().decode().split("\n")
nic_dic = dict()
next_ip_line = False
last_mac_addr = None
for line in raw_data:
if next_ip_line:
next_ip_line = False
print(line)
nic_name = last_mac_addr.split()[0]
print(nic_name)
mac_addr = last_mac_addr.split("HWaddr")[1].strip()
raw_ip_addr = line.split("inet addr:")
raw_bcast = line.split("Bcast:")
raw_netmask = line.split("Mask:")
if len(raw_ip_addr) > 1:
ip_addr = raw_ip_addr[1].split()[0]
network = raw_bcast[1].split()[0]
netmask = raw_netmask[1].split()[0]
else:
ip_addr = None
network = None
netmask = None
if mac_addr not in nic_dic:
nic_dic[mac_addr] = {'name': nic_name,
'mac': mac_addr,
'net_mask': netmask,
'network': network,
'bonding': 0,
'model': 'unknown',
'ip_address': ip_addr,
}
else:
if '%s_bonding_addr' % (mac_addr,) not in nic_dic:
random_mac_addr = '%s_bonding_addr' % (mac_addr,)
else:
random_mac_addr = '%s_bonding_addr2' % (mac_addr,)
nic_dic[random_mac_addr] = {'name': nic_name,
'mac': random_mac_addr,
'net_mask': netmask,
'network': network,
'bonding': 1,
'model': 'unknown',
'ip_address': ip_addr,
}
if "HWaddr" in line:
next_ip_line = True
last_mac_addr = line
nic_list = []
for k, v in nic_dic.items():
nic_list.append(v)
return {'nic': nic_list}
def get_disk_info():
"""
获取存储信息。
本脚本只针对ubuntu中使用sda，且只有一块硬盘的情况。
具体查看硬盘信息的命令，请根据实际情况，实际调整。
如果需要查看Raid信息，可以尝试MegaCli工具。
:return:
"""
# raw_data = subprocess.Popen("sudo hdparm -i /dev/sda | grep Model", stdout=subprocess.PIPE, shell=True)
# raw_data = raw_data.stdout.read().decode()
# data_list = raw_data.split(",")
# print("======test%s=====" % data_list)
# model = data_list[0].split("=")[1]
# sn = data_list[2].split("=")[1].strip()
model = "自定ts牌子"
sn = "1870871010997809870960869"
size_data = subprocess.Popen("sudo fdisk -l /dev/sda | grep Disk|head -1", stdout=subprocess.PIPE, shell=True)
size_data = size_data.stdout.read().decode()
size = size_data.split(":")[1].strip().split(" ")[0]
result = {'physical_disk_driver': []}
disk_dict = dict()
disk_dict["model"] = model
disk_dict["size"] = size
disk_dict["sn"] = sn
result['physical_disk_driver'].append(disk_dict)
return result
if __name__ == "__main__":
data = collect()
print(data)ER模型
亿图、Visio
curl发起连接
curl -v 远端IP:5222 -k -x “本机IP:10080”
ssh代理
ssh功能还是很强大的，提供了三个代理参数
-L参数，正向代理-R参数，反向代理-D参数，socks5代理
linux命令
HISTTIMEFORMAT=”%Y-%m-%d:%H-%M-%S:`whoami`: ”
查看Linux主机重启的命令及时间
last reboot
who -b
who -r
top
w
uptimechrome面板
请求列表
Name：资源名称
Status：HTTP状态码
Type：请求的资源MIME类型
Initiator: 发起请求的对象或进程，他可能有以下值：
- Parser（解析器）：Chrome的 HTML 解析器发起了请求（鼠标悬停显示JS脚本）
- Redirect（重定向）：HTTP重定向启动了请求
- Script（脚本）：脚本启动了请求
- Other（其他）：一些其他进程或动作发起请求，例如用户点击链接跳转到页面或在地址栏中输入网址。
Size：服务器返回的相应大小（包括头部和包体），可显示解压后大小；注意：如果网络正常，但是网页加载资源很慢，可以看下是否某些资源的size很大，导致页面加载慢。
Time：总持续时间，从请求开始到接受响应的最后一个字节
Waterfall：各请求相关活动的直观分析图
每日一句
憎恨是一种情绪，而宽容则是一种态度。
每日一句
祸兮福所倚，福兮祸所伏！
十条重拾信心的法则
1、重新梳理信心
2、坚持自我不放弃
3、学会减轻压力
4、学习充电
5、适量参加运动
6、人与人之间和谐相处
7、适时释放
8、改变形象
9、制造环境
10、另辟蹊径