Merge pull request #178 from Zalgo2462/use-dates-fix

Add the ability to split logs into dbs by subfolder, clean up the cli

Readme.md

@@ -22,7 +22,6 @@ Additional functionality is being developed and will be included soon.
 * Clone the package:
 `git clone https://github.com/ocmdev/rita.git`
 * Change into the source directory: `cd rita`
-* Make the installer executable: `chmod +x install.sh`
 * Run the installer: `sudo ./install.sh`
 * Source your .bashrc (the installer added RITA to the PATH): `source ~/.bashrc`
 * Start MongoDB: `sudo service mongod start`
@@ -36,7 +35,7 @@ To install each component of RITA by hand, [check out the instructions in the wi
 ### Configuration File
 RITA contains a yaml format configuration file.
 
-You can specify the location for the configuration file with the **-c** command line flag. If not specified, RITA will first look for the configuration in **~/.rita/config.yaml** then **/etc/rita/config.yaml**.
+You can specify the location for the configuration file with the **-c** command line flag. If not specified, RITA will look for the configuration in **/etc/rita/config.yaml**.
 
 
 ### API Keys
@@ -65,50 +64,29 @@ To obtain an API key:
       * ```bro -r pcap_to_log.pcap local "Site::local_nets += { 192.168.0.0/24 }"  "Log::default_rotation_interval = 1 day"```
 
   * **Option 2**: Install Bro and let it monitor an interface directly [[instructions](https://www.bro.org/sphinx/quickstart/)]
-      * You may wish to [compile Bro from source](https://www.bro.org/sphinx/install/install.html) for performance reasons
+      * You may wish to [compile Bro from source](https://www.bro.org/sphinx/install/install.html) for performance reasons. [This script](https://github.com/ocmdev/bro-install) can help automate the process.
       * The automated installer for RITA installs pre-compiled Bro binaries
 
 #### Importing Data Into RITA
-  * After installing, `rita` should be in your `PATH`
+  * After installing, `rita` should be in your `PATH` and the config file should be set up ready to go. Once your Bro install has collected some logs (Bro will normally rotate logs on the hour) you can run `rita import`. Alternatively, you can manually import existing logs using one of the following options:
   * **Option 1**: Import directly from the terminal (one time import)
-    * `rita import -i path/to/your/bro_logs/ -d dataset_name`
-  * **Option 2**: Set up the Bro configuration in `~/.rita/config.yaml` for repeated imports
-    * Set `LogPath` to the `path/to/your/bro_logs`
-    * Set `DBPrefix` to an identifier common to your set of logs
-    * Set up the `DirectoryMap`
-      * Logs found in folders which match the substring on the left are imported
-      into  the dataset on the right
-    * Example
-      * Say you have two sets of logs to analyze
-        * `/share/bro_logs/networkA`
-        * `/share/bro_logs/networkB`
-      * A correct Bro config section would look like
-      ```yaml
-      Bro:
-          LogPath: /share/bro_logs/
-          DBPrefix: MyCompany_
-          DirectoryMap:
-            networkA: A
-            networkB: B
-      ```
-      * This would import `/share/brologs/networkA` into `MyCompany_A` and
-      `/share/brologs/networkB` into `MyCompany_B`
-
+    * `rita import path/to/your/bro_logs/ database_name`
+  * **Option 2**: Set up the Bro configuration in `/etc/rita/config.yaml` for repeated imports
+    * Set `ImportDirectory` to the `path/to/your/bro_logs`. The default is `/opt/bro/logs`
+    * Set `DBRoot` to an identifier common to your set of logs
 
 #### Analyzing Data With RITA
   * **Option 1**: Analyze one dataset
-    * `rita analyze -d dataset_name`
-    * Ex: `rita analyze -d MyCompany_A`
+    * `rita analyze dataset_name`
+    * Ex: `rita analyze MyCompany_A`
   * **Option 2**: Analyze all imported datasets
     * `rita analyze`
 
 #### Examining Data With RITA
   * Use the **show-X** commands
   * `-H` displays human readable data
-  * `rita show-beacons -d dataset_name -H`
-  * `rita show-blacklisted -d dataset_name -H`
-
-**A link to a video tutorial will be added soon!**
+  * `rita show-beacons dataset_name -H`
+  * `rita show-blacklisted dataset_name -H`
 
 ### Getting help
 Head over to [OFTC and join #ocmdev](https://webchat.oftc.net/?channels=ocmdev) for any questions you may have.

commands/analyze.go

@@ -22,13 +22,14 @@ import (
 func init() {
 	analyzeCommand := cli.Command{
 		Name:  "analyze",
-		Usage: "Analyze imported databases, if no [database,d] flag is specified will attempt all",
+		Usage: "Analyze imported databases",
+		UsageText: "rita analyze [command options] [database]\n\n" +
+			"If no database is specified, every database will be analyzed.",
 		Flags: []cli.Flag{
-			databaseFlag,
 			configFlag,
 		},
 		Action: func(c *cli.Context) error {
-			return analyze(c.String("database"), c.String("config"))
+			return analyze(c.Args().Get(0), c.String("config"))
 		},
 	}
 

commands/commands.go

@@ -12,32 +12,25 @@ var (
 
 	// below are some prebuilt flags that get used often in various commands
 
-	// databaseFlag allows users to specify which database they'd like to use
-	databaseFlag = cli.StringFlag{
-		Name:  "database, d",
-		Usage: "execute this command against the database named `NAME`",
-		Value: "",
-	}
-
 	// threadFlag allows users to specify how many threads should be used
 	threadFlag = cli.IntFlag{
 		Name:  "threads, t",
-		Usage: "use `N` threads when executing this command",
+		Usage: "Use `N` threads when executing this command",
 		Value: runtime.NumCPU(),
 	}
 
 	// configFlag allows users to specify an alternate config file to use
 	configFlag = cli.StringFlag{
 		Name:  "config, c",
-		Usage: "use `CONFIGFILE` as configuration when running this command",
+		Usage: "Use a given `CONFIG_FILE` when running this command",
 		Value: "",
 	}
 
 	// for output we often want a human readable option which produces a nice
 	// report instead of the simple csv style output
 	humanFlag = cli.BoolFlag{
 		Name:  "human-readable, H",
-		Usage: "print a report instead of csv",
+		Usage: "Print a report instead of csv",
 	}
 
 	blSortFlag = cli.StringFlag{

commands/delete-database.go

@@ -13,19 +13,20 @@ import (
 
 func init() {
 	reset := cli.Command{
-		Name:  "delete-database",
-		Usage: "Delete an imported database",
+		Name:      "delete-database",
+		Usage:     "Delete an imported database",
+		ArgsUsage: "<database>",
 		Flags: []cli.Flag{
-			databaseFlag,
 			configFlag,
 		},
 		Action: func(c *cli.Context) error {
 			res := database.InitResources(c.String("config"))
-			if c.String("database") == "" {
-				return cli.NewExitError("Specify a database with -d", -1)
+			db := c.Args().Get(0)
+			if db == "" {
+				return cli.NewExitError("Specify a database", -1)
 			}
 
-			fmt.Println("Are you sure you want to delete database", c.String("database"), "[Y/n]")
+			fmt.Print("Are you sure you want to delete database ", db, " [y/N] ")
 
 			read := bufio.NewReader(os.Stdin)
 
@@ -36,13 +37,15 @@ func init() {
 			response = strings.ToLower(strings.TrimSpace(response))
 
 			if response == "y" || response == "yes" {
-				fmt.Println("Deleting database:", c.String("database"))
-				return res.MetaDB.DeleteDB(c.String("database"))
-			} else if response == "n" || response == "no" {
-				return cli.NewExitError("Database "+c.String("database")+" was not deleted.", 0)
+				fmt.Println("Deleting database:", db)
+				err = res.MetaDB.DeleteDB(db)
+				if err != nil {
+					return cli.NewExitError("ERROR: "+err.Error(), -1)
+				}
 			} else {
-				return cli.NewExitError("Aborted, nothing deleted.", -1)
+				return cli.NewExitError("Database "+db+" was not deleted.", 0)
 			}
+			return nil
 		},
 	}
 

commands/import.go

@@ -2,6 +2,7 @@ package commands
 
 import (
 	"fmt"
+	"path/filepath"
 
 	"github.com/ocmdev/rita/database"
 	"github.com/ocmdev/rita/parser"
@@ -12,20 +13,16 @@ import (
 func init() {
 	importCommand := cli.Command{
 		Name:  "import",
-		Usage: "Import bro logs into the database",
+		Usage: "Import bro logs into a target database",
+		UsageText: "rita import [command options] [<import directory> <database root>]\n\n" +
+			"Logs directly in <import directory> will be imported into a database" +
+			" named <database root>. Files in a subfolder of <import directory> will be imported" +
+			" into <database root>-$SUBFOLDER_NAME. <import directory>" +
+			" and <database root> will be loaded from the configuration file unless" +
+			" BOTH arguments are supplied.",
 		Flags: []cli.Flag{
 			threadFlag,
 			configFlag,
-			cli.StringFlag{
-				Name:  "import-dir, i",
-				Usage: "Import bro logs from a `directory` into a database. This overides the config file. Must be used with --database, -d",
-				Value: "",
-			},
-			cli.StringFlag{
-				Name:  "database, d",
-				Usage: "Store imported bro logs into a database with the given `name`. This overides the config file. Must be used with --import-dir, -i",
-				Value: "",
-			},
 		},
 		Action: doImport,
 	}
@@ -36,33 +33,35 @@ func init() {
 // doImport runs the importer
 func doImport(c *cli.Context) error {
 	res := database.InitResources(c.String("config"))
-	importDir := c.String("import-dir")
-	databaseName := c.String("database")
+	importDir := c.Args().Get(0)
+	targetDatabase := c.Args().Get(1)
 	threads := util.Max(c.Int("threads")/2, 1)
 
-	//one flag was set
-	if importDir != "" && databaseName == "" || importDir == "" && databaseName != "" {
-		return cli.NewExitError(
-			"Import failed.\nUse 'rita import' to import the directories "+
-				"specified in the config file or 'rita import -i [import-dir] -d [database-name]' "+
-				"to import bro logs from a given directory.", -1)
+	//check if one argument is set but not the other
+	if importDir != "" && targetDatabase == "" ||
+		importDir == "" && targetDatabase != "" {
+		return cli.NewExitError("Both <directory to import> and <database prefix> are required to override the config file.", -1)
 	}
 
-	//both flags were set
-	if importDir != "" && databaseName != "" {
-		res.Config.S.Bro.LogPath = importDir
-		res.Config.S.Bro.DBPrefix = ""
-		//Clear out the directory map and set the default database
-		res.Config.S.Bro.DirectoryMap = make(map[string]string)
-		res.Config.S.Bro.DefaultDatabase = databaseName
+	//check if the user overrode the config file
+	if importDir != "" && targetDatabase != "" {
+		//expand relative path
+		//nolint: vetshadow
+		importDir, err := filepath.Abs(importDir)
+		if err != nil {
+			return cli.NewExitError(err.Error(), -1)
+		}
+
+		res.Config.S.Bro.ImportDirectory = importDir
+		res.Config.S.Bro.DBRoot = targetDatabase
 	}
 
-	res.Log.Infof("Importing %s\n", res.Config.S.Bro.LogPath)
-	fmt.Println("[+] Importing " + res.Config.S.Bro.LogPath)
+	res.Log.Infof("Importing %s\n", res.Config.S.Bro.ImportDirectory)
+	fmt.Println("[+] Importing " + res.Config.S.Bro.ImportDirectory)
 	importer := parser.NewFSImporter(res, threads, threads)
 	datastore := parser.NewMongoDatastore(res.DB.Session, res.MetaDB,
 		res.Config.S.Bro.ImportBuffer, res.Log)
 	importer.Run(datastore)
-	res.Log.Infof("Finished importing %s\n", res.Config.S.Bro.LogPath)
+	res.Log.Infof("Finished importing %s\n", res.Config.S.Bro.ImportDirectory)
 	return nil
 }

commands/reporting.go

@@ -10,18 +10,15 @@ func init() {
 	command := cli.Command{
 
 		Name:  "html-report",
-		Usage: "Write analysis information to html output",
+		Usage: "Create an html report for an analyzed database",
+		UsageText: "rita html-report [command-options] [database]\n\n" +
+			"If no database is specified, a report will be created for every database.",
 		Flags: []cli.Flag{
 			configFlag,
-			cli.StringFlag{
-				Name:  "database, d",
-				Usage: "Specify which databases to export, otherwise will export all databases",
-				Value: "",
-			},
 		},
 		Action: func(c *cli.Context) error {
 			res := database.InitResources(c.String("config"))
-			databaseName := c.String("database")
+			databaseName := c.Args().Get(0)
 			var databases []string
 			if databaseName != "" {
 				databases = append(databases, databaseName)

commands/reset-analysis.go

@@ -13,18 +13,19 @@ import (
 
 func init() {
 	reset := cli.Command{
-		Name:  "reset-analysis",
-		Usage: "Reset analysis of one or more databases",
+		Name:      "reset-analysis",
+		Usage:     "Reset analysis of a database",
+		ArgsUsage: "<database>",
 		Flags: []cli.Flag{
-			databaseFlag,
 			configFlag,
 		},
 		Action: func(c *cli.Context) error {
 			res := database.InitResources(c.String("config"))
-			if c.String("database") == "" {
-				return cli.NewExitError("Specify a database with -d", -1)
+			db := c.Args().Get(0)
+			if db == "" {
+				return cli.NewExitError("Specify a database", -1)
 			}
-			return cleanAnalysis(c.String("database"), res)
+			return cleanAnalysis(db, res)
 		},
 	}
 

commands/show-beacons.go

@@ -13,11 +13,11 @@ import (
 
 func init() {
 	command := cli.Command{
-		Name:  "show-beacons",
-		Usage: "Print beacon information to standard out",
+		Name:      "show-beacons",
+		Usage:     "Print hosts which show signs of C2 software",
+		ArgsUsage: "<database>",
 		Flags: []cli.Flag{
 			humanFlag,
-			databaseFlag,
 			configFlag,
 		},
 		Action: showBeacons,
@@ -27,18 +27,19 @@ func init() {
 }
 
 func showBeacons(c *cli.Context) error {
-	if c.String("database") == "" {
-		return cli.NewExitError("Specify a database with -d", -1)
+	db := c.Args().Get(0)
+	if db == "" {
+		return cli.NewExitError("Specify a database", -1)
 	}
 	res := database.InitResources(c.String("config"))
-	res.DB.SelectDB(c.String("database"))
+	res.DB.SelectDB(db)
 
 	var data []beaconData.BeaconAnalysisView
 
 	ssn := res.DB.Session.Copy()
 	resultsView := beacon.GetBeaconResultsView(res, ssn, 0)
 	if resultsView == nil {
-		return cli.NewExitError("No results were found for "+c.String("database"), -1)
+		return cli.NewExitError("No results were found for "+db, -1)
 	}
 	resultsView.All(&data)
 	ssn.Close()

commands/show-bl-hostname.go

@@ -19,9 +19,9 @@ import (
 func init() {
 
 	blHostnames := cli.Command{
-		Name: "show-bl-hostnames",
+		Name:      "show-bl-hostnames",
+		ArgsUsage: "<database>",
 		Flags: []cli.Flag{
-			databaseFlag,
 			humanFlag,
 			blConnFlag,
 			blSortFlag,