编码学习

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from urllib import request
import json
from datetime import datetime, timedelta
import time

def get_data():
    url = 'https://search.51job.com/list/070200,000000,0000,00,9,99,java%25E5%25BC%2580%25E5%258F%2591,2,1.html'
    headers = {
        'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36'
    }
    req = request.Request(url, headers=headers)
    response = request.urlopen(req)
    print('===============', response.getcode(), type(response))
    if response.getcode() == 200:
        # 字节码
        data = response.read()
        print(type(data))
        # 字节码转字符串
        data = data.decode('utf-8')
        print(type(data))
        # print(data)
        with open('index.html', mode='w', encoding='utf-8') as f:
            f.write(data)

if __name__ == '__main__':
    get_data()

收集linux数据

通过python收集linux主机的cpu、硬盘、内存等

root@ubuntu16:/server/scripts# cat collect_linux_info.py 
#!/usr/bin/env python3
# -*- coding:utf-8 -*-

import subprocess

def collect():
    filter_keys = ['Manufacturer', 'Serial Number', 'Product Name', 'UUID', 'Wake-up Type']
    raw_data = {}

    for key in filter_keys:
        try:
            res = subprocess.Popen("sudo dmidecode -t system|grep '%s'" % key, stdout=subprocess.PIPE, shell=True)
            # result = res.stdout.read().decode()等于下面这两句
            result_bytes = res.stdout.read()
            result = result_bytes.decode() 
            # split分隔字符,变成列表
            data_list = result.split(':')

            if len(data_list) > 1:
                raw_data[key] = data_list[1].strip()
            else:
                raw_data[key] = ''

        except Exception as e:
            raw_data[key] = ''
        
    # 构造符合django的数据模型字段
    data = {}
    data['asset_type'] = 'server'
    data['manufacturer'] = raw_data['Manufacturer']
    data['sn'] = raw_data['Serial Number']
    data['model'] = raw_data['Product Name']
    data['uuid'] = raw_data['UUID']
    data['wake_up_type'] = raw_data['Wake-up Type']

    data.update(get_os_info())
    data.update(get_cpu_info())
    data.update(get_ram_info())
    data.update(get_nic_info())
    data.update(get_disk_info())
    return data

def get_os_info():
    """
    获取操作系统信息
    :return:
    """
    distributor = subprocess.Popen("lsb_release -a|grep 'Distributor ID'", stdout=subprocess.PIPE, shell=True)
    distributor = distributor.stdout.read().decode().split(":")
    
    release = subprocess.Popen("lsb_release -a|grep 'Description'", stdout=subprocess.PIPE, shell=True)
    release = release.stdout.read().decode().split(":")

    data_dic = {
        "os_distribution": distributor[1].strip() if len(distributor) > 1 else "",
        "os_release": release[1].strip() if len(release) > 1 else"",
        "os_type": "Linux",
    }

    return data_dic

def get_cpu_info():
    """
    获取cpu信息
    :return
    """
    raw_cmd = 'cat /proc/cpuinfo'

    raw_data = {
        'cpu_model': "%s |grep 'model name' |head -1 " % raw_cmd,
        'cpu_count': "%s | grep 'processor'|wc -l" % raw_cmd,
        'cpu_core_count': "%s | grep 'cpu cores' |awk -F : '{SUM +=$2} END {print SUM}'" % raw_cmd,
    }

    for key, cmd in raw_data.items():
        try:
            result = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
            raw_data[key] = result.stdout.read().decode().strip()
        except ValueError as e:
            print(e)
            raw_data[key] = ""


    data = {
        "cpu_count": raw_data["cpu_count"],
        "cpu_core_count": raw_data["cpu_core_count"]
    }

    cpu_model = raw_data["cpu_model"].split(":")

    return data

def get_ram_info():
    """
    获取内存信息
    :return:
    """
    raw_data = subprocess.Popen("sudo dmidecode  -t memory", stdout=subprocess.PIPE, shell=True)
    raw_list = raw_data.stdout.read().decode().split("\n")
    raw_ram_list = []
    item_list = []
    for line in raw_list:
        if line.startswith("Memory Device"):
            raw_ram_list.append(item_list)
            item_list = []
        else:
            item_list.append(line.strip())

    ram_list = []
    for item in raw_ram_list:
        item_ram_size = 0
        ram_item_to_dic = {}
        for i in item:
            data = i.split(":")
            if len(data) == 2:
                key, v = data
                if key == 'Size':
                    if v.strip() != "No Module Installed":
                        ram_item_to_dic['capacity'] = v.split()[0].strip()
                        item_ram_size = round(int(v.split()[0]))
                    else:
                        ram_item_to_dic['capacity'] = 0

                if key == 'Type':
                    ram_item_to_dic['model'] = v.strip()
             
                if key == 'Manufacturer':
                    ram_item_to_dic['manufacturer'] = v.strip()

                if key == 'Serial Number':
                    ram_item_to_dic['sn'] = v.strip()

                if key == 'Asset Tag':
                    ram_item_to_dic['asset_tag'] = v.strip()

                if key == 'Locator':
                    ram_item_to_dic['slot'] = v.strip()

            if item_ram_size == 0:
                pass
            else:
                raw_list.append(ram_item_to_dic)

    raw_total_size = subprocess.Popen("cat /proc/meminfo|grep MemTotal", stdout=subprocess.PIPE, shell=True)
    raw_total_size = raw_total_size.stdout.read().decode().split(":")
    ram_data = {'ram': ram_list}
    if len(raw_total_size) == 2:
        total_gb_size = int(raw_total_size[1].split()[0]) / 1024**2
        ram_data['ram_size'] = total_gb_size

    return ram_data

def get_nic_info():
    """
    获取网卡信息
    :return:
    """
    raw_data = subprocess.Popen("ifconfig -a", stdout=subprocess.PIPE, shell=True)

    raw_data = raw_data.stdout.read().decode().split("\n")

    nic_dic = dict()
    next_ip_line = False
    last_mac_addr = None

    for line in raw_data:
        if next_ip_line:
            next_ip_line = False
            print(line)
            nic_name = last_mac_addr.split()[0]
            print(nic_name)
            mac_addr = last_mac_addr.split("HWaddr")[1].strip()
            raw_ip_addr = line.split("inet addr:")
            raw_bcast = line.split("Bcast:")
            raw_netmask = line.split("Mask:")
            if len(raw_ip_addr) > 1:
                ip_addr = raw_ip_addr[1].split()[0]
                network = raw_bcast[1].split()[0]
                netmask = raw_netmask[1].split()[0]
            else:
                ip_addr = None
                network = None
                netmask = None
            if mac_addr not in nic_dic:
                nic_dic[mac_addr] = {'name': nic_name,
                                     'mac': mac_addr,
                                     'net_mask': netmask,
                                     'network': network,
                                     'bonding': 0,
                                     'model': 'unknown',
                                     'ip_address': ip_addr,
                                     }
            else:
                if '%s_bonding_addr' % (mac_addr,) not in nic_dic:
                    random_mac_addr = '%s_bonding_addr' % (mac_addr,)
                else:
                    random_mac_addr = '%s_bonding_addr2' % (mac_addr,)

                nic_dic[random_mac_addr] = {'name': nic_name,
                                            'mac': random_mac_addr,
                                            'net_mask': netmask,
                                            'network': network,
                                            'bonding': 1,
                                            'model': 'unknown',
                                            'ip_address': ip_addr,
                                            }

        if "HWaddr" in line:
            next_ip_line = True
            last_mac_addr = line
    nic_list = []
    for k, v in nic_dic.items():
        nic_list.append(v)

    return {'nic': nic_list}

def get_disk_info():
    """
    获取存储信息。
    本脚本只针对ubuntu中使用sda,且只有一块硬盘的情况。
    具体查看硬盘信息的命令,请根据实际情况,实际调整。
    如果需要查看Raid信息,可以尝试MegaCli工具。
    :return:
    """
   # raw_data = subprocess.Popen("sudo hdparm -i /dev/sda | grep Model", stdout=subprocess.PIPE, shell=True)
   # raw_data = raw_data.stdout.read().decode()
   # data_list = raw_data.split(",")
   # print("======test%s=====" % data_list)
   # model = data_list[0].split("=")[1]
   # sn = data_list[2].split("=")[1].strip()
    model = "自定ts牌子"
    sn = "1870871010997809870960869"

    size_data = subprocess.Popen("sudo fdisk -l /dev/sda | grep Disk|head -1", stdout=subprocess.PIPE, shell=True)
    size_data = size_data.stdout.read().decode()
    size = size_data.split(":")[1].strip().split(" ")[0]

    result = {'physical_disk_driver': []}
    disk_dict = dict()
    disk_dict["model"] = model
    disk_dict["size"] = size
    disk_dict["sn"] = sn
    result['physical_disk_driver'].append(disk_dict)

    return result

if __name__ == "__main__":
    data = collect()
    print(data)