Ludovic PLATON / IRSOM

Go to a project
Toggle navigation Toggle navigation pinning
Ludovic PLATON
c206d3de 1 parent d3c5f541

Improve performance of Featurer

Inline Side-by-side
Showing 3 changed files with 70 additions and 41 deletions
...@@ -3,6 +3,8 @@ ...@@ -3,6 +3,8 @@
3 3
4 #include <string> 4 #include <string>
5 #include <sstream> 5 #include <sstream>
6 +#include <iostream>
7 +#include <fstream>
6 8
7 #include "../multithread/buffer_producer.h" 9 #include "../multithread/buffer_producer.h"
8 #include "data_basic.h" 10 #include "data_basic.h"
...@@ -12,38 +14,60 @@ namespace data{ ...@@ -12,38 +14,60 @@ namespace data{
12 class Data_Store { 14 class Data_Store {
13 protected: 15 protected:
14 int size=0; 16 int size=0;
17 + bool write_on_add;
18 + std::string file_name;
19 + std::ofstream output;
20 + virtual void add_internal(data::Data_basic *data)=0;
15 public: 21 public:
16 - virtual ~Data_Store(){} 22 + Data_Store():write_on_add(false){}
23 + Data_Store(std::string file_name):
24 + write_on_add(true),
25 + file_name(file_name)
26 + {
27 + this->output.open(this->file_name,std::ios::trunc);
28 + }
29 +
30 + virtual ~Data_Store(){
31 + this->output.close();
32 + }
33 +
17 int getSize(){return this->size;} 34 int getSize(){return this->size;}
18 - virtual void add(data::Data_basic *data)=0; 35 +
19 - virtual multithread::Buffer<data::Data_basic*>* get(std::string pattern)=0; 36 + void add(data::Data_basic *data){
20 - void write(std::string file_name, std::string pattern){ 37 + this->add_internal(data);
21 - multithread::Buffer<data::Data_basic*> *buf; 38 + if(this->write_on_add){
22 - std::string towrite; 39 + std::string tmp = data->to_csv();
23 - std::ofstream output; 40 + this->output << tmp << std::endl;
24 - output.open(file_name, std::ios::out | std::ios::trunc);
25 - buf = this->get(pattern);
26 - int count_buf(0);
27 - while(buf->available()){
28 - if(!buf->isEmpty()){
29 - data::Data_basic* tmp = buf->pop();
30 - towrite += tmp->to_csv()+"\n";
31 - count_buf ++;
32 - delete tmp;
33 - }else{
34 - std::this_thread::sleep_for(THREAD_SLEEP);
35 - }
36 - if(count_buf == NB_ELT_BUF){
37 - output << towrite;
38 - towrite.clear();
39 - count_buf=0;
40 - }
41 - }
42 - if(count_buf>0){
43 - output << towrite;
44 } 41 }
45 - output.close();
46 } 42 }
43 + virtual multithread::Buffer<data::Data_basic*>* get(std::string pattern)=0;
44 +// void write(std::string file_name, std::string pattern){
45 +// multithread::Buffer<data::Data_basic*> *buf;
46 +// std::string towrite;
47 +// std::ofstream output;
48 +// output.open(file_name, std::ios::out | std::ios::trunc);
49 +// buf = this->get(pattern);
50 +// int count_buf(0);
51 +// while(buf->available()){
52 +// if(!buf->isEmpty()){
53 +// data::Data_basic* tmp = buf->pop();
54 +// towrite += tmp->to_csv()+"\n";
55 +// count_buf ++;
56 +// delete tmp;
57 +// }else{
58 +// std::this_thread::sleep_for(THREAD_SLEEP);
59 +// }
60 +// if(count_buf == NB_ELT_BUF){
61 +// output << towrite;
62 +// towrite.clear();
63 +// count_buf=0;
64 +// }
65 +// }
66 +// if(count_buf>0){
67 +// output << towrite;
68 +// }
69 +// output.close();
70 +// }
47 }; 71 };
48 } 72 }
49 #endif // DATA_STORE_H 73 #endif // DATA_STORE_H
......
...@@ -44,19 +44,21 @@ class Producer_map: public multithread::Buffer_producer<data::Data_basic*>{ ...@@ -44,19 +44,21 @@ class Producer_map: public multithread::Buffer_producer<data::Data_basic*>{
44 protected: 44 protected:
45 std::unordered_map<std::string,data::Data_basic*> data_map; 45 std::unordered_map<std::string,data::Data_basic*> data_map;
46 46
47 + void add_internal(data::Data_basic *data){
48 + this->data_map[data->getName()] = data;
49 + this->size ++;
50 + }
51 +
47 public: 52 public:
48 Data_Store_Map(){} 53 Data_Store_Map(){}
54 + Data_Store_Map(std::string file_name):data::Data_Store(file_name){}
49 ~Data_Store_Map(){ 55 ~Data_Store_Map(){
56 + //~data::Data_Store::~Data_Store();
50 for(auto const &kv: this->data_map){ 57 for(auto const &kv: this->data_map){
51 delete kv.second; 58 delete kv.second;
52 } 59 }
53 } 60 }
54 61
55 - void add(data::Data_basic *data){
56 - this->data_map[data->getName()] = data;
57 - this->size ++;
58 - }
59 -
60 multithread::Buffer<data::Data_basic*> *get(std::string pattern){ 62 multithread::Buffer<data::Data_basic*> *get(std::string pattern){
61 Producer_map *tmp = new Producer_map(&(this->data_map),pattern); 63 Producer_map *tmp = new Producer_map(&(this->data_map),pattern);
62 tmp->start(); 64 tmp->start();
......
...@@ -53,27 +53,30 @@ int main(int argc, char* argv[]) ...@@ -53,27 +53,30 @@ int main(int argc, char* argv[])
53 53
54 /* ************** KMER ******************/ 54 /* ************** KMER ******************/
55 feature::Feature_callable *kmer = static_cast<feature::Feature_callable*>(feature::Kmer::get_callable(3)); 55 feature::Feature_callable *kmer = static_cast<feature::Feature_callable*>(feature::Kmer::get_callable(3));
56 - tmp_ds = new data::Data_Store_Map(); 56 +// tmp_ds = new data::Data_Store_Map();
57 - ds["KMER"+std::to_string(3)] = tmp_ds; 57 + tmp_ds = new data::Data_Store_Map(output+"KMER3.txt");
58 + ds["KMER3"] = tmp_ds;
58 create_feature_maker(kmer,feature::Kmer::TYPE+std::to_string(6),tmp_ds,&re); 59 create_feature_maker(kmer,feature::Kmer::TYPE+std::to_string(6),tmp_ds,&re);
59 60
60 kmer = static_cast<feature::Feature_callable*>(feature::Kmer::get_callable(6)); 61 kmer = static_cast<feature::Feature_callable*>(feature::Kmer::get_callable(6));
61 - tmp_ds = new data::Data_Store_Map(); 62 + tmp_ds = new data::Data_Store_Map(output+"KMER6.txt");
62 - ds["KMER"+std::to_string(6)] = tmp_ds; 63 + ds["KMER6"] = tmp_ds;
63 create_feature_maker(kmer,feature::Kmer::TYPE+std::to_string(6),tmp_ds,&re); 64 create_feature_maker(kmer,feature::Kmer::TYPE+std::to_string(6),tmp_ds,&re);
64 65
65 /* *********** ORF *********************/ 66 /* *********** ORF *********************/
66 /* ORF callable */ 67 /* ORF callable */
67 68
68 feature::Feature_callable *orf = static_cast<feature::Feature_callable*>(feature::ORF::get_callable()); 69 feature::Feature_callable *orf = static_cast<feature::Feature_callable*>(feature::ORF::get_callable());
69 - tmp_ds = new data::Data_Store_Map(); 70 +// tmp_ds = new data::Data_Store_Map();
71 + tmp_ds = new data::Data_Store_Map(output+"ORF.txt");
70 ds["ORF"] = tmp_ds; 72 ds["ORF"] = tmp_ds;
71 create_feature_maker(orf,feature::ORF::TYPE,tmp_ds,&re); 73 create_feature_maker(orf,feature::ORF::TYPE,tmp_ds,&re);
72 74
73 /* *********** Codon Position ***************/ 75 /* *********** Codon Position ***************/
74 76
75 feature::Feature_callable *cp = static_cast<feature::Feature_callable*>(feature::CodonPosition::get_callable()); 77 feature::Feature_callable *cp = static_cast<feature::Feature_callable*>(feature::CodonPosition::get_callable());
76 - tmp_ds = new data::Data_Store_Map(); 78 +// tmp_ds = new data::Data_Store_Map();
79 + tmp_ds = new data::Data_Store_Map(output+"CP.txt");
77 ds["CP"] = tmp_ds; 80 ds["CP"] = tmp_ds;
78 create_feature_maker(cp,feature::CodonPosition::TYPE,tmp_ds,&re); 81 create_feature_maker(cp,feature::CodonPosition::TYPE,tmp_ds,&re);
79 82
...@@ -105,8 +108,8 @@ int main(int argc, char* argv[]) ...@@ -105,8 +108,8 @@ int main(int argc, char* argv[])
105 108
106 auto it = ds.begin(); 109 auto it = ds.begin();
107 while(it != ds.end()){ 110 while(it != ds.end()){
108 - it->second->write(output+it->first+".txt",".*"); 111 +// it->second->write(output+it->first+".txt",".*");
109 - std::cout << "File " << output+it->first+".txt" << " writted" << std::endl; 112 +// std::cout << "File " << output+it->first+".txt" << " writted" << std::endl;
110 delete it->second; 113 delete it->second;
111 ++it; 114 ++it;
112 } 115 }
......