一.npy输入数据格式
数据文件夹组织形式
——————————flower ————————rose ——————图片 ————————菊花 ——————图片 图像数据编码 data_encoder.py
import globimport os.pathimport numpy as npimport tensorflow as tffrom tensorflow.python.platform import gfile# 原始输入数据的目录，这个目录下有5个子目录，每个子目录底下保存这属于该# 类别的所有图片。INPUT_DATA = r'E:\X18301096\flower_photos'# 输出文件地址。我们将整理后的图片数据通过numpy的格式保存。OUTPUT_FILE = r'E:\X18301096\flower_photos/flower_processed_data.npy'# 测试数据和验证数据比例。VALIDATION_PERCENTAGE = 10TEST_PERCENTAGE = 10# 读取数据并将数据分割成训练数据、验证数据和测试数据。def create_image_lists(sess, testing_percentage, validation_percentage): sub_dirs = [x[0] for x in os.walk(INPUT_DATA)] is_root_dir = True # 初始化各个数据集。 training_images = [] training_labels = [] testing_images = [] testing_labels = [] validation_images = [] validation_labels = [] current_label = 0 # 读取所有的子目录。 for sub_dir in sub_dirs: if is_root_dir: is_root_dir = False continue # 获取一个子目录中所有的图片文件。 extensions = ['jpg', 'jpeg', 'JPG', 'JPEG'] file_list = [] dir_name = os.path.basename(sub_dir) for extension in extensions: file_glob = os.path.join(INPUT_DATA, dir_name, '*.' + extension) file_list.extend(glob.glob(file_glob)) if not file_list: continue print("processing:", dir_name) i = 0 # 处理图片数据。 for file_name in file_list: i += 1 # 读取并解析图片，将图片转化为299*299以方便inception-v3模型来处理。 image_raw_data = gfile.FastGFile(file_name, 'rb').read() image = tf.image.decode_jpeg(image_raw_data) if image.dtype != tf.float32: image = tf.image.convert_image_dtype(image, dtype=tf.float32) image = tf.image.resize_images(image, [299, 299]) image_value = sess.run(image) # 随机划分数据聚。 chance = np.random.randint(100) if chance < validation_percentage: validation_images.append(image_value) validation_labels.append(current_label) elif chance < (testing_percentage + validation_percentage): testing_images.append(image_value) testing_labels.append(current_label) else: training_images.append(image_value) training_labels.append(current_label) if i % 200 == 0: print(i, "images processed.") current_label += 1 # 将训练数据随机打乱以获得更好的训练效果。 state = np.random.get_state() np.random.shuffle(training_images) np.random.set_state(state) np.random.shuffle(training_labels) return np.asarray([training_images, training_labels, validation_images, validation_labels, testing_images, testing_labels])#数据处理主函数def main(): with tf.Session() as sess: processed_data = create_image_lists(sess, TEST_PERCENTAGE, VALIDATION_PERCENTAGE) # 通过numpy格式保存处理后的数据。 np.save(OUTPUT_FILE, processed_data)if __name__ == '__main__': main() 图像数据解码
data_load.py
import globimport os.pathimport numpy as npimport tensorflow as tffrom tensorflow.python.platform import gfileINPUT_DATA = ''def main(): processed_data = np.load(INPUT_DATA) training_images = processed_data[0] training_labels = processed_data[1] validation_images = processed_data[2] validation_labels = processed_data[3] testing_images = processed_data[4] testing_labels = processed_data[5] print(len(training_images),len(validation_images),len(testing_images)) print(len(training_images[0]),len(training_images[0][0])) print(training_labels) print(validation_labels) print(testing_labels)if __name__ =="__main__": main()
二.TFRecord输入数据格式
将数据写成TFRecord格式
TFRecord_write.py
import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_dataimport numpy as np#生成整数型的属性def _int64_feature(value): return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))#生成字符串型的属性def _bytes_feature(value): return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))# 将数据转化为tf.train.Example格式。def make_example(image,label,pixels): # 将图像转化成一个字符串 image_raw = image.tostring() # 将一个样例转化为Example protocol Buffer，并将所有信息写入这个数据结构 example = tf.train.Example(features=tf.train.Feature(feature={ 'pixels': _int64_feature(pixels), 'label': _int64_feature(np.argmax(labels[index])), 'image_raw': _bytes_feature(image_raw) })) return example#读取mnist训练数据mnist = input_data.read_data_sets( r"E:\MNIST_data\MNIST_data",dtype=tf.uint8,one_hot=True)images = mnist.train.images#训练数据所对应的正确答案，可以作为一个属性保存在TFRECORD中labels = mnist.train.labels#训练数据的图像分辨率，这可以作为Example中的一个属性pixels = images.shape[1]num_examples = mnist.train.num_examples#输出TFRecord文件的地址filename = r"E:\X18301096\data_processing\TFRecord\output.tfrecords"#创建一个writer 来写TFRecord文件with tf.python_io.TFRecordWriter(filename) as writer: for index in range(num_examples): example = make_example(images[index],labels[index],pixels) #将一个Example写入TFRecord文件 writer.write(example.SerializeToString())print("TFRecord训练数据已保存。")#读取mnist测试数据images_test = mnist.test.imageslabels_test = mnist.test.labelspixels_test = images_test.shape[1]num_examples_test = mnist.test.num_examples#输出包含测试数据的TFRecord文件with tf.python_io.TFRecordWriter("output_test.tfrecords") as writer: for index in range(num_examples_test): example = make_example(images_test[index],labels[index],pixels_test) writer.write(example.SerializeToString())print("TFRecord测试数据已保存！") 读取TFRecord数据格式
import tensorflow as tf#读取文件。reader= tf.TFRecordReader() #创建一个队列来维护输入文件列表filename_queue = tf.train.string_input_producer(["output.tfrecords"]) #从文件中读出一个样例_,serialized_example = reader.read(filename_queue)#解析读取的样例features = tf.parse_single_example( serialized_example, features={ 'image_raw':tf.FixedLenFeature([],tf.string), 'pixels':tf.FixedLenFeature([],tf.int64), 'label':tf.FixedLenFeature([],tf.int64) })#tf.decode_raw可以将字符串解析成图像对应的像素数组images = tf.decode_raw(features['image_raw'],tf.uint8)labels = tf.cast(features['label'],tf.int32)pixels = tf.cast(features['pixels'],tf.int32)with tf.Session() as sess: #启动多线程处理输入数据 coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess,coord=coord) for i in range(10): image,label,pixel = sess.run([images,labels,pixels]) 三.tf.dataset
在数据集框架中，每个数据集代表一个数据来源；数据可能来自一个张量，一个TFRecord文件，一个文本文件。
数据集的基本使用方法
基本步骤：
1.定义数据集的构造方法
2.定义遍历器
one_shot_iterator
initializable_iterator
3.使用get_next()方法从遍历器中读取数据张量，作为计算图其它部分的输入
import tensorflow as tfimport tempfile#从数组创建数据集input_data = [1,2,3,4,5,8]dataset = tf.data.Dataset.from_tensor_slices(input_data)#定义迭代器，用于遍历数据集iterator = dataset.make_one_shot_iterator()#get_next()返回代表一个输入数据的张量x = iterator.get_next()y = x * xwith tf.Session() as sess: for i in range(len(input_data)): print(y.eval())#读取文本文件里的数据#创建文本文件作为本例的输入with open("./test1.txt","w") as file: file.write("File1,line1.\n") file.write("File1,line2.\n")with open("./test2.txt","w") as file: file.write("File2,line1.\n") file.write("File2,line2.\n")#从文本文件创建数据集。可提供多个文件input_files= ["./test1.txt","./test2.txt"]dataset = tf.data.TextLineDataset(input_files)#定义迭代器iterator = dataset.make_one_shot_iterator()#get_next()返回一个字符串类型的张量，代表文件中的一行x = iterator.get_next()with tf.Session() as sess: for i in range(4): print(x.eval())# 解析TFRecord文件里的数据def parser(record): features = tf.parse_single_example( record, features={ 'image_raw':tf.FixedLenFeature([],tf.string), 'pixels':tf.FixedLenFeature([],tf.int64), 'label':tf.FixedLenFeature([],tf.int64) } ) decode_images = tf.decode_raw(features['image_raw'],tf.uint8) retyped_images = tf.cast(decode_images,tf.float32) images = tf.reshape(retyped_images,[784]) labels = tf.cast(features['label'],tf.int32) return images,labels#从TFRecord文件创建数据集。这里可以提供多个文件input_files = ["../output.tfrecords"]dataset = tf.data.TFRecordDataset(input_files)#map()函数表示对数据集中的每一条数据进行调用解析方法dataset = dataset.map(parser)#定义遍历数据集的迭代器iterator = dataset.make_one_shot_iterator()#读取数据，可用于进一步计算image,label = iterator.get_next()with tf.Session() as sess: for i in range(10): x,y = sess.run([image,label]) print(y)#使用initializable_iterator来动态初始化数据集# 从TFRecord文件创建数据集，具体文件路径是一个placeholder，稍后再提供具体路径。input_files = tf.placeholder(tf.string)dataset = tf.data.TFRecordDataset(input_files)dataset = dataset.map(parser)# 定义遍历dataset的initializable_iterator。iterator = dataset.make_initializable_iterator()image, label = iterator.get_next()with tf.Session() as sess: # 首先初始化iterator，并给出input_files的值。 sess.run(iterator.initializer, feed_dict={input_files: ["../output.tfrecords"]}) # 遍历所有数据一个epoch。当遍历结束时，程序会抛出OutOfRangeError。 while True: try: x, y = sess.run([image, label]) except tf.errors.OutOfRangeError: break 数据集的高层操作
import tensorflow as tf#列举输入文件# 输入数据使用本章第一节（1. TFRecord样例程序.ipynb）生成的训练和测试数据。train_files = tf.train.match_filenames_once("../output.tfrecords")test_files = tf.train.match_filenames_once("../output_test.tfrecords")#定义解析TFRecord文件的parser方法# 解析一个TFRecord的方法。def parser(record): features = tf.parse_single_example( record, features={ 'image_raw':tf.FixedLenFeature([],tf.string), 'pixels':tf.FixedLenFeature([],tf.int64), 'label':tf.FixedLenFeature([],tf.int64) }) decoded_images = tf.decode_raw(features['image_raw'],tf.uint8) retyped_images = tf.cast(decoded_images, tf.float32) images = tf.reshape(retyped_images, [784]) labels = tf.cast(features['label'],tf.int32) #pixels = tf.cast(features['pixels'],tf.int32) return images, labels#定义训练数据集image_size = 299 # 定义神经网络输入层图片的大小。batch_size = 100 # 定义组合数据batch的大小。shuffle_buffer = 10000 # 定义随机打乱数据时buffer的大小。# 定义读取训练数据的数据集。dataset = tf.data.TFRecordDataset(train_files)dataset = dataset.map(parser)# 对数据进行shuffle和batching操作。这里省略了对图像做随机调整的预处理步骤。随机打乱顺序，将数据组合成batchdataset = dataset.shuffle(shuffle_buffer).batch(batch_size)# 重复NUM_EPOCHS个epoch。NUM_EPOCHS = 10 #将数据集重复N份dataset = dataset.repeat(NUM_EPOCHS)# 定义数据集迭代器。iterator = dataset.make_initializable_iterator()image_batch, label_batch = iterator.get_next()# 定义神经网络结构和优化过程# 定义神经网络的结构以及优化过程。这里与7.3. 4小节相同。def inference(input_tensor, weights1, biases1, weights2, biases2): layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1) return tf.matmul(layer1, weights2) + biases2INPUT_NODE = 784OUTPUT_NODE = 10LAYER1_NODE = 500REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 5000weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))y = inference(image_batch, weights1, biases1, weights2, biases2)# 计算交叉熵及其平均值cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=label_batch)cross_entropy_mean = tf.reduce_mean(cross_entropy)# 损失函数的计算regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)regularaztion = regularizer(weights1) + regularizer(weights2)loss = cross_entropy_mean + regularaztion# 优化损失函数train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss)#定义测试用数据集# 定义测试用的Dataset。test_dataset = tf.data.TFRecordDataset(test_files)test_dataset = test_dataset.map(parser)test_dataset = test_dataset.batch(batch_size)# 定义测试数据上的迭代器。test_iterator = test_dataset.make_initializable_iterator()test_image_batch, test_label_batch = test_iterator.get_next()# 定义测试数据上的预测结果。test_logit = inference(test_image_batch, weights1, biases1, weights2, biases2)predictions = tf.argmax(test_logit, axis=-1, output_type=tf.int32)# 声明会话并运行神经网络的优化过程。with tf.Session() as sess: # 初始化变量。 sess.run((tf.global_variables_initializer(), tf.local_variables_initializer())) # 初始化训练数据的迭代器。 sess.run(iterator.initializer) # 循环进行训练，直到数据集完成输入、抛出OutOfRangeError错误。 while True: try: sess.run(train_step) except tf.errors.OutOfRangeError: break test_results = [] test_labels = [] # 初始化测试数据的迭代器。 sess.run(test_iterator.initializer) # 获取预测结果。 while True: try: pred, label = sess.run([predictions, test_label_batch]) print(pred.shape) test_results.extend(pred) test_labels.extend(label) except tf.errors.OutOfRangeError: break# 计算准确率correct = [float(y == y_) for (y, y_) in zip(test_results, test_labels)]accuracy = sum(correct) / len(correct)print("Test accuracy is:", accuracy) 四.多线程输入数据处理框架
import tensorflow as tf#创建文件列表，通过文件列表创建输入文件队列，读取文件为本章第一节创建的文件files = tf.train.match_filenames_once("./output.tfrecords")filename_queue = tf.train.string_input_producer(files, shuffle=False)# 解析TFRecord文件里的数据# 读取文件。reader = tf.TFRecordReader()_,serialized_example = reader.read(filename_queue)# 解析读取的样例。features = tf.parse_single_example( serialized_example, features={ 'image_raw':tf.FixedLenFeature([],tf.string), 'pixels':tf.FixedLenFeature([],tf.int64), 'label':tf.FixedLenFeature([],tf.int64) })#从原始图像数据解析出像素矩阵，并根据图像尺寸还原图像decoded_images = tf.decode_raw(features['image_raw'],tf.uint8)retyped_images = tf.cast(decoded_images, tf.float32)labels = tf.cast(features['label'],tf.int32)#pixels = tf.cast(features['pixels'],tf.int32)images = tf.reshape(retyped_images, [784])#将文件以100个为一组打包。min_after_dequeue = 10000batch_size = 100capacity = min_after_dequeue + 3 * batch_sizeimage_batch, label_batch = tf.train.shuffle_batch([images, labels], batch_size=batch_size, capacity=capacity, min_after_dequeue=min_after_dequeue)#训练模型def inference(input_tensor, weights1, biases1, weights2, biases2): layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1) return tf.matmul(layer1, weights2) + biases2# 模型相关的参数INPUT_NODE = 784OUTPUT_NODE = 10LAYER1_NODE = 500REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 5000weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))y = inference(image_batch, weights1, biases1, weights2, biases2)# 计算交叉熵及其平均值cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=label_batch)cross_entropy_mean = tf.reduce_mean(cross_entropy)# 损失函数的计算regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)regularaztion = regularizer(weights1) + regularizer(weights2)loss = cross_entropy_mean + regularaztion# 优化损失函数train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss)# 初始化会话，并开始训练过程。with tf.Session() as sess: # tf.global_variables_initializer().run() sess.run((tf.global_variables_initializer(), tf.local_variables_initializer())) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) # 循环的训练神经网络。 for i in range(TRAINING_STEPS): if i % 1000 == 0: print("After %d training step(s), loss is %g " % (i, sess.run(loss))) sess.run(train_step) coord.request_stop() coord.join(threads)