如何使用DMA在一个线程中同时写入不同磁盘上的多个文件？

Question

我使用aio在一个线程的不同磁盘上写入多个文件。当我使用缓冲写入时，IO处理是并发的。但CPU负载非常高。当我用DIRECT标志打开文件时，IO处理不是并发的。

如何使用DMA在一个线程中同时写入不同磁盘上的多个文件？

#include <malloc.h> 
#include <stdio.h> 
#include <string.h> 
#include <iostream> 
#include <sstream> 
#include <inttypes.h> 

#include <unistd.h> 
#include <fcntl.h> 
#include <sys/syscall.h> 
#include <linux/aio_abi.h> 

using namespace std; 

long double timeDiff(timespec start, timespec end) { 
const long double s = start.tv_sec + start.tv_nsec * 1.0e-9; 
const long double e = end.tv_sec + end.tv_nsec * 1.0e-9; 
return e - s; 
} 

// nr: maximum number of requests that can simultaneously reside in the context. 
inline int io_setup(unsigned nr, aio_context_t *ctxp) { 
return syscall(__NR_io_setup, nr, ctxp); 
} 

inline int io_destroy(aio_context_t ctx) { 
return syscall(__NR_io_destroy, ctx); 
} 

// Every I/O request that is submitted to 
inline int io_submit(aio_context_t ctx, long nr, struct iocb **iocbpp) { 
return syscall(__NR_io_submit, ctx, nr, iocbpp); 
} 

// For every completed I/O request kernel creates an io_event structure. 
// minimal number of events one wants to get. 
// maximum number of events one wants to get. 
inline int io_getevents(aio_context_t ctx, long min_nr, long max_nr, 
    struct io_event *events, struct timespec *timeout) { 
return syscall(__NR_io_getevents, ctx, min_nr, max_nr, events, timeout); 
} 

int main(int argc, char *argv[]) { 

// prepare data 
const unsigned int kAlignment = 4096; 
const long data_size = 1600 * 1024 * 12/8; 
//const long data_size = 2448 * 1344 * 12/8; 
void * data = memalign(kAlignment, data_size); 
memset(data, 0, data_size); 
//for (int i = 0; i < data_size; ++i) 
// data[i] = 'A'; 

// prepare fd 
//const int file_num = 3; 
const int file_num = 2; 
int fd_arr[file_num]; 
for (int i = 0; i < file_num; ++i) { 
    ostringstream filename; 
    if (i == 0) { 
     //filename << "/data/test"; 
     filename << "/test"; 
    } else { 
     filename << "/data" << i << "/test"; 
    } 
    //filename << "/data/test" << i; 
    int fd = open(filename.str().c_str(), O_WRONLY | O_NONBLOCK | O_CREAT | O_DIRECT | O_APPEND, 0644); 
    //int fd = open(filename.str().c_str(), O_WRONLY | O_NONBLOCK | O_CREAT | O_DIRECT, 0644); 
    //int fd = open(filename.str().c_str(), O_WRONLY | O_NONBLOCK | O_CREAT, 0644); 
    if (fd < 0) { 
     perror("open"); 
     return -1; 
    } 
    fd_arr[i] = fd; 
} 

aio_context_t ctx; 
struct io_event events[file_num]; 
int ret; 
ctx = 0; 

ret = io_setup(1000, &ctx); 
if (ret < 0) { 
    perror("io_setup"); 
    return -1; 
} 

struct iocb cbs[file_num]; 
for (int i = 0; i < file_num; ++i) { 
    memset(&cbs[i], 0, sizeof(cbs[i])); 
} 
struct iocb * cbs_pointer[file_num]; 
for (int i = 0; i < file_num; ++i) { 
    /* setup I/O control block */ 
    cbs_pointer[i] = &cbs[i]; 
    cbs[i].aio_fildes = fd_arr[i]; 
    cbs[i].aio_lio_opcode = IOCB_CMD_PWRITE; // IOCV_CMD 
    cbs[i].aio_nbytes = data_size; 
} 

timespec tStart, tCurr; 
clock_gettime(CLOCK_REALTIME, &tStart); 

const int frame_num = 10000; 
for (int k = 0; k < frame_num; ++k) { 

    for (int i = 0; i < file_num; ++i) { 
     /* setup I/O control block */ 
     cbs[i].aio_buf = (uint64_t)data; 
     //cbs[i].aio_offset = k * data_size; 
    } 

    ret = io_submit(ctx, file_num, cbs_pointer); 
    if (ret < 0) { 
     perror("io_submit"); 
     return -1; 
    } 

    /* get reply */ 
    ret = io_getevents(ctx, file_num, file_num, events, NULL); 
    //printf("events: %d, k: %d\n", ret, k); 
} 

clock_gettime(CLOCK_REALTIME, &tCurr); 
cout << "frame: " << frame_num << " time: " << timeDiff(tStart, tCurr) << endl; 

ret = io_destroy(ctx); 
if (ret < 0) { 
    perror("io_destroy"); 
    return -1; 
} 

// close fd 
for (int i = 0; i < file_num; ++i) { 
    fsync(fd_arr[i]); 
    close(fd_arr[i]); 
} 
return 0; 
} 

Answer 1

当且仅当正在写入的物理盘区已分配到盘上时，Linux才能使写入实际上异步。否则，它必须先取得一个互斥锁并先进行分配，这样所有内容才会变成同步的。

请注意，截断文件到一个新的长度通常不会实际分配底层范围。您需要先预先写好内容。此后，重写现在将执行相同的范围异步并因此变为并发。

正如你可能正在收集的那样，Linux上的异步文件I/O并不好，尽管它随着时间的推移不断变好。 Windows或FreeBSD有很多优秀的实现。即使OS X并不可怕。改用其中的任何一种。