Merge pull request #214 from voutcn/refactor

1.2.3-beta

CHANGELOG.md

@@ -1,3 +1,7 @@
+### 1.2.3-beta / 2019-05-12 PST
+-   Refactored sequence readers
+-   Fixed a bug in SDBG building of large k-mer sizes
+
 ### 1.2.2-beta / 2019-04-16 PST
 -   Automatically detect POPCNT/BMI2 and select the correct megahit_core binary
 

CMakeLists.txt

@@ -18,6 +18,7 @@ FILE(GLOB_RECURSE LCASM_SOURCE "src/localasm/*.cpp")
 FILE(GLOB_RECURSE IDBA_SOURCE "src/idba/*.cpp")
 FILE(GLOB_RECURSE SDBG_SOURCE "src/sdbg/*.cpp")
 FILE(GLOB_RECURSE CX1_SOURCE "src/cx1/*.cpp")
+FILE(GLOB_RECURSE SEQ_SOURCE "src/sequence/*.cpp")
 FILE(GLOB_RECURSE TOOLKIT_SOURCE "src/tools/*.cpp")
 
 LIST(APPEND OTHER_SOURCE
@@ -27,8 +28,7 @@ LIST(APPEND OTHER_SOURCE
         src/main_iterate.cpp
         src/main_local_assemble.cpp
         src/main_sdbg_build.cpp
-        src/options_description.cpp
-        src/sequence_manager.cpp
+        src/utils/options_description.cpp
         )
 
 if (STATIC_BUILD)
@@ -42,7 +42,8 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DXXH_INLINE_ALL -D__STDC_FORMAT_MACROS")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-unused-function")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprefetch-loop-arrays -funroll-loops")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__XFILE__='\"$(subst ${CMAKE_SOURCE_DIR}/,,$(abspath $<))\"'")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__XROOT__='\"${CMAKE_SOURCE_DIR}/src\"'")
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__XFILE__='\"$(subst ${CMAKE_SOURCE_DIR}/,,$(abspath $<))\"'")
 
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ZLIB_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
 
@@ -57,10 +58,10 @@ set(CMAKE_CXX_FLAGS_DEBUG "-g -ggdb -O1")
 
 message(STATUS "Build type: ${CMAKE_BUILD_TYPE}: ${CMAKE_CXX_FLAGS}")
 
-add_executable(megahit_core ${OTHER_SOURCE} ${ASMBL_SOURCE} ${IDBA_SOURCE} ${SDBG_SOURCE} ${LCASM_SOURCE}
+add_executable(megahit_core ${OTHER_SOURCE} ${ASMBL_SOURCE} ${IDBA_SOURCE} ${SDBG_SOURCE} ${LCASM_SOURCE} ${SEQ_SOURCE}
         ${CX1_SOURCE} ${TOOLKIT_SOURCE})
 add_executable(megahit_core_no_hw_accel ${OTHER_SOURCE} ${ASMBL_SOURCE} ${IDBA_SOURCE} ${SDBG_SOURCE} ${LCASM_SOURCE}
-        ${CX1_SOURCE} ${TOOLKIT_SOURCE})
+        ${SEQ_SOURCE} ${CX1_SOURCE} ${TOOLKIT_SOURCE})
 set_target_properties(megahit_core PROPERTIES COMPILE_FLAGS "-mbmi2 -DUSE_BMI2 -mpopcnt")
 
 if (STATIC_BUILD)
@@ -82,9 +83,10 @@ set(TEST_DATA ${CMAKE_SOURCE_DIR}/test_data)
 
 add_custom_target(
         simple_test
-        COMMAND ./megahit --test -t 2 --keep-tmp-files
-        COMMAND ./megahit --test -t 2 --keep-tmp-files --no-hw-accel
-        COMMAND ./megahit --test -t 2 --keep-tmp-files --kmin-1pass
+        COMMAND ./megahit --test -t 2
+        COMMAND ./megahit --test -t 2 --no-hw-accel
+        COMMAND ./megahit --test -t 2 --kmin-1pass
+        COMMAND rm -rf test-random && python3 ../test_data/generate_random_fasta.py > random.fa && ./megahit -r random.fa --k-list 255 --min-count 1 -o test-random
 )
 
 add_dependencies(megahit megahit_core megahit_core_no_hw_accel)

src/assembly/bubble_remover.h

@@ -6,7 +6,7 @@
 #define MEGAHIT_BUBBLE_REMOVER_H
 
 #include <cstdint>
-#include "histgram.h"
+#include "utils/histgram.h"
 #include "unitig_graph.h"
 #include "contig_writer.h"
 

src/assembly/contig_stat.h

@@ -6,8 +6,8 @@
 #define MEGAHIT_CONTIG_STAT_H
 
 #include <map>
-#include "utils.h"
-#include "histgram.h"
+#include "utils/utils.h"
+#include "utils/histgram.h"
 #include "unitig_graph.h"
 
 using ContigStat = std::map<std::string, uint64_t>;
@@ -37,7 +37,7 @@ inline ContigStat CalcAndPrintStat(UnitigGraph &graph, bool print = true, bool c
   };
 
   if (print) {
-    xinfo("")
+    xinfo("");
     for (auto &kv: stat) {
       xinfoc("%s: %lu, ", kv.first.c_str(), kv.second);
     }

src/assembly/contig_writer.h

@@ -8,7 +8,7 @@
 #include <string>
 #include <cstdint>
 #include <cassert>
-#include "histgram.h"
+#include "utils/histgram.h"
 
 class UnitigGraph;
 

src/assembly/sdbg_pruning.cpp

@@ -31,8 +31,8 @@
 #include <queue>
 
 #include "kmlib/kmbitvector.h"
-#include "utils.h"
-#include "histgram.h"
+#include "utils/utils.h"
+#include "utils/histgram.h"
 
 using std::vector;
 using std::string;

src/assembly/unitig_graph.cpp

@@ -5,7 +5,7 @@
 #include "unitig_graph.h"
 #include <omp.h>
 #include <mutex>
-#include "utils.h"
+#include "utils/utils.h"
 
 UnitigGraph::UnitigGraph(SDBG *sdbg)
     : sdbg_(sdbg), adapter_impl_(this), sudo_adapter_impl_(this) {

src/cx1/cx1.h

@@ -29,7 +29,7 @@
 #include <vector>
 #include <algorithm>
 
-#include "utils.h"
+#include "utils/utils.h"
 
 /**
  * @brief    an CX1 engine

src/cx1/cx1_kmer_count.cpp

@@ -27,12 +27,12 @@
 #include <omp.h>
 #include <mutex>
 
-#include "safe_alloc_open-inl.h"
-#include "sequence/kseq.h"
-#include "utils.h"
+#include "utils/safe_alloc_open-inl.h"
+#include "sequence/readers/kseq.h"
+#include "utils/utils.h"
 #include "sequence/kmer.h"
-#include "packed_reads.h"
-#include "read_lib_functions-inl.h"
+#include "sequence/packed_reads.h"
+#include "sequence/read_lib_functions-inl.h"
 
 #include "sorting.h"
 
@@ -109,27 +109,19 @@ void read_input_prepare(count_global_t &globals) { // num_items_, num_cpu_thread
         num_reads += num_ass_seq;
     }
 
-  globals.package.ReserveSequences(num_reads);
-  globals.package.ReserveBases(num_bases);
+    globals.package.ReserveSequences(num_reads);
+    globals.package.ReserveBases(num_bases);
 
     ReadBinaryLibs(globals.read_lib_file, globals.package, globals.lib_info, is_reverse);
 
     if (globals.assist_seq_file != "") {
-        SequenceManager seq_manager;
-        seq_manager.set_file_type(SequenceManager::kFastxReads);
-        seq_manager.set_file(globals.assist_seq_file);
-
-        bool reverse_read = true;
-        bool append_to_package = true;
-        bool trimN = false;
-
-        seq_manager.ReadShortReads(1ULL << 60, 1ULL << 60, append_to_package, reverse_read, trimN);
-        seq_manager.clear();
+      FastxReader reader({globals.assist_seq_file});
+      reader.ReadAll(&globals.package, is_reverse);
     }
 
     globals.package.BuildIndex();
     globals.max_read_length = globals.package.MaxSequenceLength();
-    globals.num_reads = globals.package.size();
+    globals.num_reads = globals.package.Size();
 
     xinfo("%ld reads, %d max read length\n", globals.num_reads, globals.max_read_length);
 
@@ -616,7 +608,6 @@ void post_proc(count_global_t &globals) {
     int64_t num_candidate_reads = 0;
     int64_t num_has_tips = 0;
     FILE *candidate_file = xfopen((globals.output_prefix + ".cand").c_str(), "wb");
-    SequenceManager seq_manager(&globals.package);
 
     for (int64_t i = 0; i < globals.num_reads; ++i) {
         auto first = globals.first_0_out[i].v.load(std::memory_order::memory_order_acquire);
@@ -627,7 +618,7 @@ void post_proc(count_global_t &globals) {
 
             if (last > first) {
                 ++num_candidate_reads;
-                seq_manager.WriteBinarySequences(candidate_file, false, i, i);
+                WriteBinarySequences(globals.package, candidate_file, i, i);
             }
         }
     }

src/cx1/cx1_kmer_count.h

@@ -27,9 +27,9 @@
 #include <mutex>
 #include "definitions.h"
 #include "cx1.h"
-#include "edge_io.h"
+#include "sequence/readers/edge_io.h"
 #include "sequence/sequence_package.h"
-#include "lib_info.h"
+#include "sequence/lib_info.h"
 #include "utils/atomic_wrapper.h"
 
 struct count_opt_t {

src/cx1/cx1_read2sdbg.h

@@ -29,7 +29,7 @@
 #include "sdbg/sdbg_writer.h"
 #include "kmlib/kmbitvector.h"
 #include "sequence/sequence_package.h"
-#include "lib_info.h"
+#include "sequence/lib_info.h"
 #include "definitions.h"
 
 struct read2sdbg_opt_t {

src/cx1/cx1_read2sdbg_s1.cpp

@@ -26,13 +26,13 @@
 #include <omp.h>
 #include <mutex>
 
-#include "safe_alloc_open-inl.h"
-#include "sequence/kseq.h"
-#include "utils.h"
+#include "utils/safe_alloc_open-inl.h"
+#include "sequence/readers/kseq.h"
+#include "utils/utils.h"
 #include "sequence/kmer.h"
-#include "packed_reads.h"
+#include "sequence/packed_reads.h"
 #include "sequence/sequence_package.h"
-#include "read_lib_functions-inl.h"
+#include "sequence/read_lib_functions-inl.h"
 
 #include "sorting.h"
 // helping functions
@@ -118,22 +118,12 @@ void s1_read_input_prepare(read2sdbg_global_t &globals) {
     globals.max_read_length = globals.package.MaxSequenceLength();
 
     if (globals.assist_seq_file != "") {
-        SequenceManager seq_manager;
-        seq_manager.set_readlib_type(SequenceManager::kSingle);
-        seq_manager.set_file_type(SequenceManager::kFastxReads);
-        seq_manager.set_file(globals.assist_seq_file);
-        seq_manager.set_package(&globals.package);
-
-        bool reverse_read = true;
-        bool append_to_package = true;
-        bool trimN = false;
-
-        seq_manager.ReadShortReads(1LL << 60, 1LL << 60, append_to_package, reverse_read, trimN);
-        seq_manager.clear();
+      FastxReader reader({globals.assist_seq_file});
+      reader.ReadAll(&globals.package, is_reverse);
     }
 
-  globals.package.BuildIndex();
-    globals.num_reads = globals.package.size();
+    globals.package.BuildIndex();
+    globals.num_reads = globals.package.Size();
 
     xinfo("%ld reads, %d max read length, %lld total bases\n", globals.num_reads, globals.max_read_length,
           globals.package.BaseCount());

src/cx1/cx1_read2sdbg_s2.cpp

@@ -26,11 +26,11 @@
 #include <parallel/algorithm>
 #include <mutex>
 
-#include "utils.h"
+#include "utils/utils.h"
 #include "sequence/kmer.h"
-#include "safe_alloc_open-inl.h"
-#include "packed_reads.h"
-#include "read_lib_functions-inl.h"
+#include "utils/safe_alloc_open-inl.h"
+#include "sequence/packed_reads.h"
+#include "sequence/read_lib_functions-inl.h"
 
 #include "sorting.h"