Merge pull request #153 from jaebeom-kim/master

Syncmer

Author: jaebeom-kim

README.md

@@ -31,6 +31,19 @@ Please cite: [Kim J, Steinegger M. Metabuli: sensitive and specific metagenomic
 
 
 ---
+### Update after latest release
+- ***Syncmer is introduced:*** half database size, double classification speed.
+- Older Metabuli versions are not compatible with the new database format.
+- New GTDB R226 database [here](https://hulk.mmseqs.com/jaebeom/gtdb226db/)
+  - Species representative genomes with checkm2 completeness > 70% and contamination < 5% are included.
+  - 129,671 species out of 143,614 species in GTDB R226 are included.
+- New pre-built at [https://mmseqs.com/metabuli](https://mmseqs.com/metabuli)
+  
+### Update in v1.1.1
+- `--validate-input` option added for FASTA/Q validation.
+- `--validate-db` option added for database validation.
+- `classifiedRefiner` module added to refine classification results.
+  
 ### Update in v1.1.0
 - Fix errors in v1.0.9
 - Custom DB creation became easier

src/CMakeLists.txt

@@ -1,6 +1,7 @@
 include_directories(commons)
 include_directories(workflow)
 include_directories(util)
+include_directories(benchmark)
 include_directories(../lib/prodigal)
 include_directories(../lib/fasta_validator)
 include_directories(../lib/fastq_utils)
@@ -9,11 +10,13 @@ add_subdirectory(commons)
 add_subdirectory(util)
 add_subdirectory(version)
 add_subdirectory(workflow)
+add_subdirectory(benchmark)
 
 add_executable(metabuli
         ${commons_source_files}
         ${util_source_files}
         ${workflow_source_files}
+        ${benchmark_source_files}
         metabuli.cpp
         LocalCommandDeclarations.h 
         util/filter_by_genus.cpp

src/LocalCommandDeclarations.h

@@ -10,7 +10,6 @@ extern int grade(int argc, const char **argv, const Command& command);
 extern int gradeByCladeSize(int argc, const char **argv, const Command& command);
 extern int seqHeader2TaxId(int argc, const char **argv, const Command& command);
 extern int addToLibrary(int argc, const char **argv, const Command& command);
-extern int applyThreshold(int argc, const char **argv, const Command& command);
 extern int binning2report(int argc, const char **argv, const Command& command);
 extern int filterByGenus(int argc, const char **argv, const Command& command);
 extern int databaseReport(int argc, const char **argv, const Command& command);
@@ -27,5 +26,9 @@ extern int editNames(int argc, const char **argv, const Command& command);
 extern int createnewtaxalist(int argc, const char **argv, const Command& command);
 extern int classifiedRefiner(int argc, const char **argv, const Command& command);
 extern int validateDatabase(int argc, const char **argv, const Command& command);
+extern int printDeltaIdx(int argc, const char **argv, const Command& command); 
+extern int makeBenchmarkSet(int argc, const char **argv, const Command &command);
+extern int makeQuerySet(int argc, const char **argv, const Command &command);
+extern int makeVirusBenchmarkSet(int argc, const char **argv, const Command &command);
 
 #endif //ADCLASSIFIER2_LOCALCOMMANDDECLARATIONS_H

src/MetabuliBase.cpp

@@ -52,6 +52,13 @@ std::vector<Command> metabuliCommands = {
                 "<diffIdx>",
                 CITATION_SPACEPHARER,
                 {{"Differential index", DbType::ACCESS_MODE_OUTPUT, DbType::NEED_DATA, &DbValidator::empty}}},
+        {"printDeltaIdx", printDeltaIdx, &localPar.expand_diffidx, COMMAND_EXPERT,
+                "Print k-mers stored in delta index",
+               nullptr,
+                "Jaebeom Kim <[email protected]>",
+                "<deltaIdx>",
+                CITATION_SPACEPHARER,
+                {{"deltaIdx", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile}}},
         {"printInfo", printInfo, &localPar.printInfo, COMMAND_EXPERT,
                 "Print k-mer information",
                nullptr,
@@ -196,7 +203,32 @@ std::vector<Command> metabuliCommands = {
                 "Jaebeom Kim <[email protected]>",
                 "<i: database directory>",
                 CITATION_SPACEPHARER,
-                {{"database directory", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory}}}
+                {{"database directory", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory}}},
+        {"maketestsets", makeBenchmarkSet, &localPar.makeBenchmarkSet, COMMAND_EXPERT,
+                "Create test sets for benchmarking",
+                nullptr,
+                "Jaebeom Kim <[email protected]}",
+                "<i: Assembly accessions> <i: Taxonomy dump>",
+                CITATION_SPACEPHARER,
+                {{"Assembly accessions", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile},
+                 {"Taxonomy dump", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory}}},
+        {"makeInclusionTestQueries", makeQuerySet, &localPar.makeBenchmarkSet, COMMAND_EXPERT,
+                "Create query sets for inclusion tests.",
+                nullptr,
+                "Jaebeom Kim <[email protected]}",
+                "<i: Assembly accessions> <i: Taxonomy dump>",
+                CITATION_SPACEPHARER,
+                {{"Assembly accessions", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile},
+                 {"Taxonomy dump", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory}}},
+        {"make-virus-benchmark-set", makeVirusBenchmarkSet, &localPar.makeBenchmarkSet, COMMAND_EXPERT,
+                "Create test sets for benchmarking virus classification",
+                nullptr,
+                "Jaebeom Kim <[email protected]}",
+                "<i: assembly accessions> <i: taxonomy dump> <i: assembly accession 2 taxid>",
+                CITATION_SPACEPHARER,
+                {{"Assembly accessions", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile},
+                 {"Taxonomy dump", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::directory},
+                 {"Assembly accession to taxid mapping", DbType::ACCESS_MODE_INPUT, DbType::NEED_DATA, &DbValidator::flatfile}}}
 };
 
 std::vector<KmerThreshold> externalThreshold = {};

src/benchmark/CMakeLists.txt

@@ -0,0 +1,5 @@
+set(benchmark_source_files
+        benchmark/makeInclusionQuerySet.cpp
+        benchmark/makeBenchmarkSet.cpp
+        benchmark/makeVirusBenchmarkSet.cpp
+        PARENT_SCOPE)