merge dev into main

README.md

@@ -4,16 +4,38 @@ This Docker image will discover APIs in your GitHub account by scanning for open
 
 
 
-## Building
+## Quickstart
 
-You can build the image yourself by cloning the repository and using the following docker command:
+First, clone this repo and build the scanner's image:
 
 ```bash
 git clone git@github.com:FireTail-io/github-api-discovery.git
 cd github-api-discovery
 docker build --rm -t firetail-io/github-api-discovery:latest -f build_setup/Dockerfile . --target runtime
 ```
 
+Make a copy of the provided [config-example.yml](./config-example.yml) and call it `config.yml`, then edit it for your use case.
+
+```bash
+cp config-example.yml config.yml
+open config.yml
+```
+
+Running the image requires two environment variables to be set:
+
+- `GITHUB_TOKEN`, [a classic GitHub personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token-classic).
+- `FIRETAIL_APP_TOKEN`, [a FireTail app token](https://www.firetail.io/docs/create-app-token).
+
+Find a full list of environment variables under [Environment Variables](#environment-variables).
+
+Once you have created a classic GitHub personal access token and a FireTail app token, you can run the scanner image:
+
+```bash
+export GITHUB_TOKEN=YOUR_GITHUB_TOKEN
+export FIRETAIL_APP_TOKEN=YOUR_FIRETAIL_APP_TOKEN
+docker run --rm -e GITHUB_TOKEN=${GITHUB_TOKEN} -e FIRETAIL_APP_TOKEN=${FIRETAIL_APP_TOKEN} --mount type=bind,source="$(pwd)"/config.yml,target=/config.yml,readonly firetail-io/github-api-discovery:latest
+```
+
 
 
 ## Tests
@@ -33,19 +55,7 @@ docker run --rm --entrypoint cat firetail-io/github-api-discovery:test-golang co
 
 
 
-## Running
-
-Running the image requires two environment variables, `GITHUB_TOKEN` and `FIRETAIL_APP_TOKEN`. You can find a full list of environment variables used by the scanner below.
-
-The scanner also requires a config file to determine the organisations, users and repositories to scan. You can find an example at [config-example.yml](./config-example.yml). 
-
-Copy [config-example.yml](./config-example.yml) to `config.yml` and adjust it to your use case, then run the image using the following docker command:
-
-```bash
-export GITHUB_TOKEN=YOUR_GITHUB_TOKEN
-export FIRETAIL_APP_TOKEN=YOUR_FIRETAIL_APP_TOKEN
-docker run --rm -e GITHUB_TOKEN=${GITHUB_TOKEN} -e FIRETAIL_APP_TOKEN=${FIRETAIL_APP_TOKEN} --mount type=bind,source="$(pwd)"/config.yml,target=/config.yml,readonly firetail-io/github-api-discovery:latest
-```
+## Environment Variables
 
 | Variable Name        | Description                                                  | Required? | Default                                          |
 | -------------------- | ------------------------------------------------------------ | --------- | ------------------------------------------------ |

analysers/golang/analyse.go

@@ -35,7 +35,7 @@ func analyse(filePath string, fileContents string) (map[string]string, map[strin
 	imports := map[string]string{}
 	for _, parsedImport := range parsedFile.Imports {
 		// parsedImport.Path.Value includes quotes on either end (e.g. "net/http"); these indexes strip them.
-		importPath := parsedImport.Path.Value[1:len(parsedImport.Path.Value)-1]
+		importPath := parsedImport.Path.Value[1 : len(parsedImport.Path.Value)-1]
 		if parsedImport.Name != nil {
 			imports[importPath] = parsedImport.Name.Name
 		} else {
@@ -51,23 +51,33 @@ func analyse(filePath string, fileContents string) (map[string]string, map[strin
 	if packageIdentifier, ok := importedSupportedFrameworks["net/http"]; ok {
 		netHttpPathsSlice := analyseNetHTTP(parsedFile, packageIdentifier)
 
-		netHttpPathsMap := map[string]map[string]map[string]map[string]string{}
+		netHttpPathsMap := map[string]map[string]map[string]map[string]map[string]string{}
 		for _, path := range netHttpPathsSlice {
-			netHttpPathsMap[path] = map[string]map[string]map[string]string{
+			responses := map[string]map[string]map[string]string{
 				"responses": {
 					"default": {
 						"description": "Discovered via static analysis",
 					},
 				},
 			}
+			netHttpPathsMap[path] = map[string]map[string]map[string]map[string]string{
+				"get":     responses,
+				"post":    responses,
+				"put":     responses,
+				"patch":   responses,
+				"delete":  responses,
+				"head":    responses,
+				"options": responses,
+				"trace":   responses,
+			}
 		}
 
 		// Only make an appspec if there's at least one path detected
 		if len(netHttpPathsMap) > 0 {
-			openapiSpecs["static-analysis:net/http:" + filePath] = map[string]interface{}{
+			openapiSpecs["static-analysis:net/http:"+filePath] = map[string]interface{}{
 				"openapi": "3.0.0",
-				"info": map[string]string{"title": "Static Analysis - Golang net/http"},
-				"paths": netHttpPathsMap,
+				"info":    map[string]string{"title": "Static Analysis - Golang net/http"},
+				"paths":   netHttpPathsMap,
 			}
 		}
 	}

analysers/golang/analyse_test.go

@@ -27,17 +27,28 @@ func main() {
 	imports, openapiSpecs, err := analyse(fileName, fileContents)
 	assert.Nil(t, err)
 	assert.Equal(t, map[string]string{"net/http": "http"}, imports)
+
+	responses := map[string]map[string]map[string]string{
+		"responses": {
+			"default": {
+				"description": "Discovered via static analysis",
+			},
+		},
+	}
 	assert.Equal(t, map[string]interface{}{
-			"static-analysis:net/http:net_http_hello_world.go": map[string]interface{}{
+		"static-analysis:net/http:net_http_hello_world.go": map[string]interface{}{
 			"openapi": "3.0.0",
-			"info": map[string]string{"title": "Static Analysis - Golang net/http"},
-			"paths": map[string]map[string]map[string]map[string]string{
+			"info":    map[string]string{"title": "Static Analysis - Golang net/http"},
+			"paths": map[string]map[string]map[string]map[string]map[string]string{
 				"/hello": {
-					"responses": {
-						"default": {
-							"description": "Discovered via static analysis",
-						},
-					},
+					"get":     responses,
+					"post":    responses,
+					"put":     responses,
+					"patch":   responses,
+					"delete":  responses,
+					"head":    responses,
+					"options": responses,
+					"trace":   responses,
 				}},
 		},
 	}, openapiSpecs)
@@ -64,17 +75,28 @@ func main() {
 	imports, openapiSpecs, err := analyse(fileName, fileContents)
 	assert.Nil(t, err)
 	assert.Equal(t, map[string]string{"net/http": "nethttp"}, imports)
+
+	responses := map[string]map[string]map[string]string{
+		"responses": {
+			"default": {
+				"description": "Discovered via static analysis",
+			},
+		},
+	}
 	assert.Equal(t, map[string]interface{}{
-			"static-analysis:net/http:net_http_hello_world.go": map[string]interface{}{
+		"static-analysis:net/http:net_http_hello_world.go": map[string]interface{}{
 			"openapi": "3.0.0",
-			"info": map[string]string{"title": "Static Analysis - Golang net/http"},
-			"paths": map[string]map[string]map[string]map[string]string{
+			"info":    map[string]string{"title": "Static Analysis - Golang net/http"},
+			"paths": map[string]map[string]map[string]map[string]map[string]string{
 				"/hello": {
-					"responses": {
-						"default": {
-							"description": "Discovered via static analysis",
-						},
-					},
+					"get":     responses,
+					"post":    responses,
+					"put":     responses,
+					"patch":   responses,
+					"delete":  responses,
+					"head":    responses,
+					"options": responses,
+					"trace":   responses,
 				},
 			},
 		},
@@ -90,4 +112,4 @@ func TestAnalyseGolangNotGoFile(t *testing.T) {
 	assert.Equal(t, "net_http_hello_world.go:1:1: expected 'package', found '{'", err.Error())
 	assert.Equal(t, map[string]string{}, imports)
 	assert.Equal(t, map[string]interface{}{}, openapiSpecs)
-}
+}

build_setup/Dockerfile

@@ -29,6 +29,8 @@ FROM build-python as test-python
 WORKDIR /github-api-discovery
 COPY setup.cfg /github-api-discovery/setup.cfg
 RUN python3 -m pip install pytest pytest-cov
+COPY tests/requirements.txt /github-api-discovery/tests/requirements.txt
+RUN python3 -m pip install -r /github-api-discovery/tests/requirements.txt
 COPY tests/ /github-api-discovery/tests/
 RUN pytest --cov /github-api-discovery --cov-report=xml:coverage.xml -vv -x
 
@@ -47,6 +49,8 @@ RUN rm -rf /build_setup
 
 FROM build-python-lambda as test-python-lambda
 RUN python3 -m pip install pytest pytest-cov
+COPY tests/requirements.txt /github-api-discovery/tests/requirements.txt
+RUN python3 -m pip install -r /github-api-discovery/tests/requirements.txt
 COPY tests/ ${LAMBDA_TASK_ROOT}/tests
 RUN PYTHONPATH=${LAMBDA_TASK_ROOT} pytest --cov ${LAMBDA_TASK_ROOT} --cov-report=xml:coverage.xml -vv -x 
 

build_setup/requirements.txt

@@ -1,6 +1,7 @@
-PyGithub
-openapi-spec-validator
-pyyaml
-prance
-tree_sitter
-dacite
+openapi-spec-validator==0.6.0
+prance==23.6.21.0
+pyyaml==6.0.1
+dacite==1.8.1
+tree_sitter==0.20.2
+PyGithub==1.59.1
+jsonschema==4.19.0

config-example.yml

@@ -1,8 +1,9 @@
 # List organisations to scan their repositories
 organisations: # default []
   example-organisation:
-    # Under each org, you can skip public, internal, archived or fork repositories
+    # Under each org, you can skip public, private, internal, archived or fork repositories
     skip_public_repositories: False # default False
+    skip_private_repositories: False # default False
     skip_internal_repositories: False # default False
     skip_archived_repositories: False # default False
     skip_forks: False # default False
@@ -19,5 +20,5 @@ users: # default []
 # List individual repositories to include or exclude them explicitly from scanning.
 # Has higher prescedence than scanning via users or orgs.
 repositories: # default []
-  example-user/example-repository: exclude # default "exclude"
-  example-organisation/example-repository: include # default "exclude"
+  example-user/example-repository: exclude
+  example-organisation/example-repository: include

src/config.py

@@ -4,6 +4,7 @@
 from github.Repository import Repository as GithubRepository
 
 
+@dataclass
 class AccountConfig(ABC):
     skip_public_repositories: bool = False
     skip_archived_repositories: bool = False
@@ -23,39 +24,39 @@ def skip_repo(self, repository: GithubRepository) -> bool:
 
 
 @dataclass
-class OrgConfig(AccountConfig):
-    skip_internal_repositories: bool = False
+class UserConfig(AccountConfig):
+    skip_private_repositories: bool = False
 
     def skip_repo(self, repository: GithubRepository) -> bool:
         super_skip = super().skip_repo(repository)
         if super_skip:
             return True
 
-        if self.skip_internal_repositories and repository.visibility == "internal":
+        if self.skip_private_repositories and repository.visibility == "private":
             return True
 
         return False
 
 
 @dataclass
-class UserConfig(AccountConfig):
-    skip_private_repositories: bool = False
+class OrgConfig(UserConfig):
+    skip_internal_repositories: bool = False
 
     def skip_repo(self, repository: GithubRepository) -> bool:
         super_skip = super().skip_repo(repository)
         if super_skip:
             return True
 
-        if self.skip_private_repositories and repository.visibility == "private":
+        if self.skip_internal_repositories and repository.visibility == "internal":
             return True
 
         return False
 
 
 @dataclass
 class Config:
-    organisations: dict[str, OrgConfig | None] | list[str] | None = field(default_factory=dict[str, OrgConfig])
-    users: dict[str, UserConfig | None] | list[str] | None = field(default_factory=dict[str, UserConfig])
+    organisations: dict[str, OrgConfig | None] | list[str] | None = field(default_factory=dict[str, OrgConfig | None])
+    users: dict[str, UserConfig | None] | list[str] | None = field(default_factory=dict[str, UserConfig | None])
     repositories: dict[str, str] | None = field(default_factory=dict)
 
     def __post_init__(self):
@@ -66,18 +67,11 @@ def __post_init__(self):
             }
         elif type(self.organisations) == list:
             self.organisations = {organisation: OrgConfig() for organisation in self.organisations}
-        elif self.organisations is None:
-            self.organisations = {}
 
         if type(self.users) == dict:
             self.users = {user: config if config is not None else UserConfig() for user, config in self.users.items()}
         elif type(self.users) == list:
             self.users = {user: UserConfig() for user in self.users}
-        elif self.users is None:
-            self.users = {}
-
-        if self.repositories is None:
-            self.repositories = {}
 
     def skip_repo(self, repository: GithubRepository) -> bool:
-        return self.repositories.get(repository.full_name) != "include"
+        return self.repositories.get(repository.full_name) == "exclude"

src/openapi/validation.py

@@ -25,18 +25,18 @@ def parse_resolve_and_validate_openapi_spec(file_path: str, get_file_contents: C
     # First check it's a valid JSON/YAML file before passing it over to Prance
     if file_path.endswith(".json"):
         try:
-            json.loads(get_file_contents())
+            file_contents = json.loads(get_file_contents())
         except:  # noqa: E722
             return None
 
     elif file_path.endswith((".yaml", ".yml")):
         try:
-            yaml.safe_load(get_file_contents())
+            file_contents = yaml.safe_load(get_file_contents())
         except:  # noqa: E722
             return None
 
     else:
         return None
 
     # If it was a valid JSON/YAML file, we can give it to Prance to load
-    return resolve_and_validate_openapi_spec(get_file_contents())
+    return resolve_and_validate_openapi_spec(yaml.dump(file_contents))