# 词典
dictionary = ['one ', 'two ', 'three ', 'four ', 'five ', 'big ', 'small ', 'light ']
# 计数器
counter = [0 for i in range(len(dictionary))]

def build_file(file_name):
    file = open(file_name, 'w')
    file_size = os.path.getsize(file_name)
    if file_size <= oneG :
        # 文件大小小于1G
        while file_size <= oneG:
            file.writelines(getWords())
            file_size = os.path.getsize(file_name)

def getWords():
    list = []
    for i in range(random.randint(1, 2)):
        index = random.randint(0, len(dictionary) - 1)
        counter[index] += 1
        list.append(dictionary[index])
    list.append('\n')
    return list

std::vector<std::map<std::string, int>> dirc_loop;

std::mutex m;

int getFileSize(std::string file)

{

if (file.empty())

return 0;

struct stat filestat;

if (stat(file.c_str(), &filestat) < 0) {

perror("error: ");

}

return filestat.st_size;

}

int fileExist(std::string file)

{

if (file.empty())

return 0;

struct stat filestat;

if (stat(file.c_str(), &filestat) < 0) {

perror("error: ");

return 0;

}

printf("file exist\n");

return 1;

}

int readFile(std::string fileName, size_t file_offsize, size_t end)

{

std::map<std::string, int> dirc;

std::fstream file;

file.open(fileName, std::ios::in);

if (!file.is_open())

return -1;

// 设置文件指针位置

file.seekg(file_offsize, std::ios::beg);

// 查找新的完整单词开始位置

if (file_offsize != 0) {

int last_c = file.seekg(-1, std::ios::cur).get();

if (last_c != ' ' && last_c != '\n') {

while (true) {

int c = file.get();

if (c == ' ' || c == '\n')

break;

}

}

}

printf("read start\n");

// 开始读取

std::string line;

while (file >> line) {

if (dirc.find(line) != dirc.end())

dirc[line] += 1;

else

dirc[line] = 1;

size_t endPos = file.tellg();

if (endPos + 1 >= end)

break;

}

m.lock();

dirc_loop.push_back(dirc);

m.unlock();

return 0;

}

int main()

{

printf("start\n");

std::fstream file;

std::string fileName = "D:\\PyCharm\\workspace\\Project\\计操\\small_file"; // 文件名

if (!fileExist(fileName))

return 0;

printf("file size: %d\n", getFileSize(fileName));

int parts = 4;   //设置线程数

int dater = getFileSize(fileName) / parts;

std::vector<std::thread> t_loop;

auto start_time = std::chrono::system_clock::now();

for (int i = 0; i < parts; i++) t_loop.push_back(std::thread(readFile, fileName, i * dater, (i + 1) * dater));

for (int i = 0; i < parts; i++) t_loop[i].join();

std::cout << "SPENT TIME: " << std::chrono::duration_cast<std::chrono::microseconds>((std::chrono::system_clock::now() - start_time)).count() << " 微秒" << std::endl;

printf("read over\n");

// 汇总结果

for (int i = 0; i < dirc_loop.size(); i++)

{

if (i == 0) continue;

for (auto d : dirc_loop[i]) {

if (dirc_loop[0].find(d.first) != dirc_loop[0].end()) {

dirc_loop[0][d.first] += d.second;

}

else {

dirc_loop[0][d.first] = d.second;

}

}

}

// 输出

for (auto i : dirc_loop[0])

{

printf("%s %d\n", i.first.c_str(), i.second);

}

}