物品分类 cifar10 & cifar100 学习笔记

使用keras实现。比较简单。这里就是做个思路的总结和代码的注释。

首先我们要准备好环境和数据集。 emmmmmmmm，https://keras.io/examples/cifar10_cnn/ 上面的链接是keras官网的教学代码，十分简洁。我也是这个。

首先学习一下cifar10的数据集：https://www.cs.toronto.edu/~kriz/cifar.html

数据集中有如下几个文件： batches.meta data_batch_1 data_batch_2 data_batch_3 data_batch_4 data_batch_5 test_batch

这些文件都是 生成的。所以我们用人家官网的函数进行拆包，拆开后是一个字典。 可以用for in看看都有那些数据项。

拆包函数： def unpickle(file): import pickle with open(file, 'rb') as fo: dict = pickle.load(fo, encoding='bytes') return dict

用for in 看看发现，其中batches.meta里边包含了： b'num_cases_per_batch' #数据的多少 b'label_names' #标签的名字 b'num_vis' #*** 这三个数据项，注意这些关键字都是bytes类型的，并不是字符串。

我们做分类其实就只是需要数据和标签就好，标签也是一个整型数而已。 比如我们这次要做一个10个物品的一个分类，那么我们的标签的数值范围就是0-9喽， 作为入门的卷积神经网络，我们只需要把最后的输出层调成10个神经元就好。 所以，这里的label_name可以作为一个映射，最后我们预测时方便自己阅读而已。

data_batch_1-5是训练集，每个文件有10000张图片，大小是1000032323 总共5个文件合并到一起就是5000032323

test_batch是预测集，有10000张图片，大小是100003232*3

下面为代码：（其中我没有使用keras自带的那个加载数据的函数，为了更好的理解，所以自己写了加载数据的函数）

首先是加载数据标签的代码：class_name.py

file = "data/batches.meta"


def unpickle(file):
    import pickle
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict

def readClassName():
    result = unpickle(file)
    label_name = result[b'label_names']
    return label_name

if __name__=="__main__":
    result = unpickle(file)
    label_name = result[b'label_names']
    print(label_name)

emmmm,没啥可解释的。 下面是加载数据的代码：load_data.py

import numpy as np
import cv2

def unpickle(file):
    import pickle
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict

#将一张3072的一维图片调整成32*32*3的三维图片格式
def arraytoimage(image):
    r = image[0:1024]
    g = image[1024:2048]
    b = image[2048:]

    r = np.reshape(r, (32, 32))
    g = np.reshape(g, (32, 32))
    b = np.reshape(b, (32, 32))
    image = cv2.merge([r,g,b])
    return image

#读取训练集的一个文件
def load_one_train_batch(no):
    file_path = "data/data_batch_"
    file_path+=str(no)

    file = unpickle(file_path)
    data = file[b'data']
    label = file[b'labels']

    result = []
    for i in data:
        #由于一直图片是以rgb的形式存放的，是一个以为的向量3072
        #我们需要将其调整成32*32*3这样的三维向量，因为我们需要把这个作为卷积层的输入，这样才能使用
        #特征提取器，即卷积核来提取特征，从而识别物品
        image = arraytoimage(i)
        result.append(image)

    return result,label

#读取测试集
def load_test_batch():
    file_path = "data/test_batch"
    file = unpickle(file_path)
    data = file[b'data']
    label = file[b'labels']

    result = []
    for image in data:
        image = arraytoimage(image)
        result.append(image)

    return result, label

#读取数据，里边包含对训练集的读取和对测试集的读取
def load_data():
    #load tarin data
    train_datas = []
    train_labels = []
    for i in range(1,6):
        data,label = load_one_train_batch(i)
        train_datas.append(data)
        train_labels.append(label)
    #由于放回回来的数据类型为list不能用，所以调整为np.array类型的
    train_datas = np.reshape(train_datas,(50000,32,32,3))
    train_labels = np.reshape(train_labels,(50000,1))

    #load test data
    test_data,test_label = load_test_batch()
    test_data = np.reshape(test_data,(10000,32,32,3))
    test_label = np.reshape(test_label,(10000,1))

    return (train_datas,train_labels),(test_data,test_label)

if __name__=="__main__":
    (train_datas,train_labels),(test_data,test_label) = load_data()

emmm，这样数据读取就搞定了，就剩下使用keras定义卷积神经网络，并进行训练就好了。

代码：cifar10.py

import tensorflow as tf
from keras.datasets import cifar10
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense,Dropout,Activation,Flatten,Conv2D,MaxPooling2D,ZeroPadding2D
from keras.utils.training_utils import multi_gpu_model   #导入keras多GPU函数
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from class_name import readClassName
import load_data

np.random.seed(10)

model_path = "model/cifarCnnModel_1.h5"

#读取类别的名字到label_names
label_names = readClassName()

#根据图片实际的结果和预测的结果，画出来从idx开始的10个图像
def plot_images_lables_prediction(images,lables,prediction,idx,num=10):
    flg = plt.gcf()
    flg.set_size_inches(12,14)
    if num>25:num=25
    for i in range(0,num):
        ax=plt.subplot(5,5,i+1)
        ax.imshow(images[idx],cmap='binary')

        title = str(i)+','+str(label_names[lables[i][0]],encoding="utf8")
        if len(prediction)>0:
            title+="=>"+str(label_names[prediction[i]],encoding="utf8")
        ax.set_title(title,fontsize=10)
        ax.set_xticks([])
        ax.set_yticks([])
        idx+=1
    plt.show()

#显示训练的记录
def show_train_history(train_history,train,validation):
    plt.plot(train_history.history[train])
    plt.plot(train_history.history[validation])
    plt.title("Train history")
    plt.ylabel(train)
    plt.xlabel('Epoch')
    plt.legend(['train','validation'],loc='upper left')
    plt.show()


if __name__=="__main__":
    #官方函数的读取数据方法
    #(x_img_train,y_lable_train),(x_img_test,y_lable_test)=cifar10.load_data()
    #根据我们的加载数据的方法
    (x_img_train, y_lable_train), (x_img_test, y_lable_test) = load_data.load_data()

    print(type(x_img_train))
    print(np.shape(x_img_train))
    print(type(y_lable_train))
    print(np.shape(y_lable_train))

    '''
    <class 'numpy.ndarray'>
    (50000, 32, 32, 3)
    <class 'numpy.ndarray'>
    (50000, 1)
    '''

    print('train:',len(x_img_train))
    print('test:',len(x_img_test))

    y_lable_train_OneHot = np_utils.to_categorical(y_lable_train)
    y_lable_test_OneHot = np_utils.to_categorical(y_lable_test)

    '''
    将训练和测试用的标签从一个0-9的整数，转化为一个长度为10的向量，那么50000个训练的标签
    最终就是50000*10的一个ndarry
        <class 'numpy.ndarray'>
        (50000, 10)
    '''

    #二值化
    x_img_train_normalize = x_img_train.astype('float32') / 255
    x_img_test_normalize = x_img_test.astype('float32') / 255

    #建立模型
    model = Sequential()

    #建立卷积层1，池化层1
    model.add(Conv2D(filters=32,
                     kernel_size=(3,3),
                     input_shape=(32,32,3),
                     activation='relu',
                     padding='same'))

    #假如Dropout，防止过拟合
    model.add(Dropout(0.25))

    #建立卷积层2
    model.add(Conv2D(filters=32,
                     kernel_size=(3,3),
                     activation='relu',
                     padding='same'))

    #建立池化层1
    model.add(MaxPooling2D(pool_size=(2,2)))

    #建立平坦层
    model.add(Flatten())

    # 假如Dropout，防止过拟合
    model.add(Dropout(0.25))

    #建立隐藏层1
    model.add(Dense(2500,activation='relu'))

    #建立输出层
    model.add(Dense(10,activation='softmax'))

    #print(model.summary())

    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])

    try:
        model.load_weights(model_path)
        print("加载模型成功!")
    except:
        print("加载模型失败!")

    #输出对模型的总结
    #model.summary()

    train_history = model.fit(x=x_img_train_normalize,  #训练数据
                              y=y_lable_train_OneHot,   #训练数据的OneHot标签
                              validation_split=0.2,     #validation_split用于在没有提供验证集的时候，按一定比例从训练集中取出一部分作为验证集
                              epochs=1,                 #训练的轮数
                              batch_size=300,           #https://blog.csdn.net/Mr_Brooks/article/details/80277114
                              verbose=2)                #https://blog.csdn.net/my_kingdom/article/details/84313683

    #保存权重模型
    model.save_weights(model_path)
    print("model save success!")

    #显示训练的记录
    show_train_history(train_history,'acc','val_acc')
    show_train_history(train_history, 'loss', 'val_loss')

    #使用测试集对训练的模型进行评估
    scores = model.evaluate(x_img_test_normalize,
                            y_lable_test_OneHot,
                            verbose=0)

    print('accuracy=', scores[1])

    #对测试集进行预测
    prediction = model.predict_classes(x_img_test_normalize)

    #展示出0-10这10个预测结果
    plot_images_lables_prediction(x_img_test,y_lable_test,prediction,0,10)

以上就是cifar10 的全部内容

下面cifar100也是一样的写法 cifar100所提供的数据的标签有一个细标签，一个粗标签，然而我并不像关注那个粗标签，直接把细标签作为了图像的label。

class_name.py:

def unpickle(file):
    import pickle
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict


def readClassName():
    file_path = "data/cifar-100-python/meta"
    dic = unpickle(file_path)
    '''
    b'fine_label_names'
    b'coarse_label_names'
    '''
    fine_label_names = dic[b'fine_label_names']
    coarse_label_names = dic[b'coarse_label_names']
    return fine_label_names,coarse_label_names

if __name__=="__main__":
    f,c = readClassName()
    print(f)
    print(c)

load_data.py

import numpy as np
import cv2
import class_name

image_no = 10

def arraytoimage(image):
    r = image[0:1024]
    g = image[1024:2048]
    b = image[2048:]

    r = np.reshape(r, (32, 32))
    g = np.reshape(g, (32, 32))
    b = np.reshape(b, (32, 32))
    image = cv2.merge([r,g,b])
    return image

def unpickle(file):
    import pickle
    with open(file, 'rb') as fo:
        dict = pickle.load(fo, encoding='bytes')
    return dict

def load_train_data():
    file_path = "data/cifar-100-python/train"
    train_data = unpickle(file_path)
    '''
    b'filenames'
    b'batch_label'
    b'fine_labels'
    b'coarse_labels'
    b'data'
    '''

    # 因为fine这个细标签就是了，他的范围为0-99的，我直接做一个中文对应就好
    f, c = class_name.readClassName()
    data = train_data[b'data']
    fine_labels = train_data[b'fine_labels']

    datas = []
    labels = []
    for i in range(len(data)):
        datas.append(arraytoimage(data[i]))
        labels.append(fine_labels[i])

    datas = np.reshape(datas, (50000, 32, 32, 3))
    labels = np.reshape(labels, (50000, 1))
    return datas, labels

def load_test_data():
    file_path = "data/cifar-100-python/test"
    train_data = unpickle(file_path)

    # 因为fine这个细标签就是了，他的范围为0-99的，我直接做一个中文对应就好
    f, c = class_name.readClassName()
    data = train_data[b'data']
    fine_labels = train_data[b'fine_labels']

    datas = []
    labels = []
    for i in range(len(data)):
        datas.append(arraytoimage(data[i]))
        labels.append(fine_labels[i])

    # print(np.shape(datas))
    # print(np.shape(labels))
    datas = np.reshape(datas, (10000, 32, 32, 3))
    labels = np.reshape(labels, (10000, 1))
    return datas, labels

def load_data():
    train_data,train_label=load_train_data()
    test_data,test_label=load_test_data()
    return (train_data,train_label),(test_data,test_label)


if __name__=="__main__":
    (x_img_train, y_lable_train), (x_img_test, y_lable_test) = load_data()

cifar100.py:

import tensorflow as tf
from keras.datasets import cifar10
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense,Dropout,Activation,Flatten,Conv2D,MaxPooling2D,ZeroPadding2D
from keras.utils.training_utils import multi_gpu_model   #导入keras多GPU函数
import os
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from class_name import readClassName
import load_data

np.random.seed(10)

model_path = "model/cifar100CnnModel_1.h5"

fine_label_names,coarse_label_names = readClassName()

def plot_images_lables_prediction(images,lables,prediction,idx,num=10):
    flg = plt.gcf()
    flg.set_size_inches(12,14)
    if num>25:num=25
    for i in range(0,num):
        ax=plt.subplot(5,5,i+1)
        ax.imshow(images[idx],cmap='binary')
        label_name = str(fine_label_names[lables[i][0]], encoding="utf8")
        title = str(i)+','+label_name
        if len(prediction)>0:
            title+="=>"+label_name
        ax.set_title(title,fontsize=10)
        ax.set_xticks([])
        ax.set_yticks([])
        idx+=1
    plt.show()

def show_train_history(train_history,train,validation):
    plt.plot(train_history.history[train])
    plt.plot(train_history.history[validation])
    plt.title("Train history")
    plt.ylabel(train)
    plt.xlabel('Epoch')
    plt.legend(['train','validation'],loc='upper left')
    plt.show()


if __name__=="__main__":
    (x_img_train, y_lable_train), (x_img_test, y_lable_test) = load_data.load_data()

    print(type(x_img_train))
    print(np.shape(x_img_train))
    print(type(y_lable_train))
    print(np.shape(y_lable_train))

    '''
    <class 'numpy.ndarray'>
    (50000, 32, 32, 3)
    <class 'numpy.ndarray'>
    (50000, 1)
    '''

    print('train:',len(x_img_train))
    print('test:',len(x_img_test))

    y_lable_train_OneHot = np_utils.to_categorical(y_lable_train)
    y_lable_test_OneHot = np_utils.to_categorical(y_lable_test)

    #二值化
    x_img_train_normalize = x_img_train.astype('float32') / 255
    x_img_test_normalize = x_img_test.astype('float32') / 255

    #建立模型
    model = Sequential()

    #建立卷积层1，池化层1
    model.add(Conv2D(filters=32,
                     kernel_size=(3,3),
                     input_shape=(32,32,3),
                     activation='relu',
                     padding='same'))

    #假如Dropout，防止过拟合
    model.add(Dropout(0.25))

    #建立卷积层2
    model.add(Conv2D(filters=32,
                     kernel_size=(3,3),
                     activation='relu',
                     padding='same'))

    #建立池化层1
    model.add(MaxPooling2D(pool_size=(2,2)))

    #建立平坦层
    model.add(Flatten())

    # 假如Dropout，防止过拟合
    model.add(Dropout(0.25))

    #建立隐藏层1
    model.add(Dense(2500,activation='relu'))

    #建立输出层
    model.add(Dense(100,activation='softmax'))

    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])

    try:
        model.load_weights(model_path)
        print("加载模型成功!")
    except:
        print("加载模型失败!")

    model.summary()

    train_history = model.fit(x=x_img_train_normalize,
                              y=y_lable_train_OneHot,
                              validation_split=0.2,
                              epochs=2,
                              batch_size=300,
                              verbose=2)

    model.save_weights(model_path)
    print("model save success!")


    show_train_history(train_history,'acc','val_acc')
    show_train_history(train_history, 'loss', 'val_loss')

    scores = model.evaluate(x_img_test_normalize,
                            y_lable_test_OneHot,
                            verbose=0)

    print('accuracy=', scores[1])

    prediction = model.predict_classes(x_img_test_normalize)

    plot_images_lables_prediction(x_img_test,y_lable_test,prediction,0,10)