Skip to content

Commit

Permalink
Update postman metadata (#3852)
Browse files Browse the repository at this point in the history 
* Updated Postman metadata fields to contain location uniqueness and took out the unused fields of global_id, field_name, and variable_type.
* Disabled body scanning for now since the only body that is scanned is the currently selected radio button but secrets can still be saved in the other unselected radio button options.
* Updated link generation for more accuracy.
* Updated tests to not use global constant.
  • Loading branch information
@casey-tran
casey-tran authored Jan 30, 2025
1 parent b6b00bb commit 9ecaf07
Show file tree
Hide file tree
Showing 7 changed files with 519 additions and 360 deletions.
10 changes: 10 additions & 0 deletions pkg/output/plain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,16 @@ func structToMap(obj any) (m map[string]map[string]any, err error) {
return
}
err = json.Unmarshal(data, &m)
// Due to PostmanLocationType protobuf field being an enum, we want to be able to assign the string value of the enum to the field without needing to create another Protobuf field.
// To have the "UNKNOWN_POSTMAN = 0" value be assigned correctly to the field, we need to check if the Postman workspace ID is filled since every secret in the Postman source
// should have a valid workspace ID and the 0 value is considered nil for integers.
if m["Postman"]["workspace_uuid"] != nil {
if m["Postman"]["location_type"] == nil {
m["Postman"]["location_type"] = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN.String()
} else {
m["Postman"]["location_type"] = obj.(*source_metadatapb.MetaData_Postman).Postman.LocationType.String()
}
}
return
}

Expand Down
705 changes: 408 additions & 297 deletions pkg/pb/source_metadatapb/source_metadata.pb.go
View file

Large diffs are not rendered by default.

6 changes: 1 addition & 5 deletions pkg/pb/source_metadatapb/source_metadata.pb.validate.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

108 changes: 64 additions & 44 deletions pkg/sources/postman/postman.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
const (
SourceType = sourcespb.SourceType_SOURCE_TYPE_POSTMAN
LINK_BASE_URL = "https://go.postman.co/"
GLOBAL_TYPE = "globals"
ENVIRONMENT_TYPE = "environment"
AUTH_TYPE = "authorization"
REQUEST_TYPE = "request"
Expand Down Expand Up @@ -147,7 +146,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
if err = json.Unmarshal(contents, &env); err != nil {
return err
}
s.scanVariableData(ctx, chunksChan, Metadata{EnvironmentName: env.ID, fromLocal: true, Link: envPath}, env)
s.scanVariableData(ctx, chunksChan, Metadata{EnvironmentID: env.ID, EnvironmentName: env.Name, fromLocal: true, Link: envPath, LocationType: source_metadatapb.PostmanLocationType_ENVIRONMENT_VARIABLE}, env)
}

// Scan local workspaces
Expand Down Expand Up @@ -230,7 +229,9 @@ func (s *Source) scanLocalWorkspace(ctx context.Context, chunksChan chan *source

for _, environment := range workspace.EnvironmentsRaw {
metadata.Link = strings.TrimSuffix(path.Base(filePath), path.Ext(filePath)) + "/environments/" + environment.ID + ".json"
metadata.LocationType = source_metadatapb.PostmanLocationType_ENVIRONMENT_VARIABLE
s.scanVariableData(ctx, chunksChan, metadata, environment)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}
for _, collection := range workspace.CollectionsRaw {
metadata.Link = strings.TrimSuffix(path.Base(filePath), path.Ext(filePath)) + "/collections/" + collection.Info.PostmanID + ".json"
Expand Down Expand Up @@ -265,13 +266,20 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
metadata.Link = LINK_BASE_URL + "environments/" + envID.UUID
metadata.FullID = envVars.ID
metadata.EnvironmentID = envID.UUID
metadata.EnvironmentName = envVars.Name

ctx.Logger().V(2).Info("scanning environment vars", "environment_uuid", metadata.FullID)
for _, word := range strings.Split(envVars.Name, " ") {
s.attemptToAddKeyword(word)
}

metadata.LocationType = source_metadatapb.PostmanLocationType_ENVIRONMENT_VARIABLE
s.scanVariableData(ctx, chunksChan, metadata, envVars)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
metadata.Type = ""
metadata.Link = ""
metadata.FullID = ""
metadata.EnvironmentID = ""
metadata.EnvironmentName = ""
ctx.Logger().V(2).Info("finished scanning environment vars", "environment_uuid", metadata.FullID)
}
ctx.Logger().V(2).Info("finished scanning environments")
Expand Down Expand Up @@ -305,11 +313,13 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch
metadata.Link = LINK_BASE_URL + COLLECTION_TYPE + "/" + metadata.FullID
}

metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_VARIABLE
// variables must be scanned first before drilling down into the folders and events
// because we need to pick up the substitutions from the top level collection variables
s.scanVariableData(ctx, chunksChan, metadata, VariableData{
KeyValues: collection.Variables,
})
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN

for _, event := range collection.Events {
s.scanEvent(ctx, chunksChan, metadata, event)
Expand Down Expand Up @@ -345,6 +355,7 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c

// check if there are any requests in the folder
if item.Request.Method != "" {
metadata.FolderName = strings.Replace(metadata.FolderName, (" > " + item.Name), "", -1)
metadata.RequestID = item.ID
metadata.RequestName = item.Name
metadata.Type = REQUEST_TYPE
Expand All @@ -368,8 +379,16 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c
s.scanEvent(ctx, chunksChan, metadata, event)
}

if metadata.RequestID != "" {
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_AUTHORIZATION
} else if metadata.FolderID != "" {
metadata.LocationType = source_metadatapb.PostmanLocationType_FOLDER_AUTHORIZATION
} else if metadata.CollectionInfo.UID != "" {
metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION
}
// an auth all by its lonesome could be inherited to subfolders and requests
s.scanAuth(ctx, chunksChan, metadata, item.Auth, item.Request.URL)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, event Event) {
Expand All @@ -378,15 +397,24 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk,

// Prep direct links. Ignore updating link if it's a local JSON file
if !metadata.fromLocal {
metadata.Link = LINK_BASE_URL + metadata.Type + "/" + metadata.FullID
metadata.Link = LINK_BASE_URL + (strings.Replace(metadata.Type, " > event", "", -1)) + "/" + metadata.FullID
if event.Listen == "prerequest" {
metadata.Link += "?tab=pre-request-scripts"
} else {
metadata.Link += "?tab=tests"
}
}

if strings.Contains(metadata.Type, REQUEST_TYPE) {
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_SCRIPT
} else if strings.Contains(metadata.Type, FOLDER_TYPE) {
metadata.LocationType = source_metadatapb.PostmanLocationType_FOLDER_SCRIPT
} else if strings.Contains(metadata.Type, COLLECTION_TYPE) {
metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT
}

s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, data)), metadata)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, auth Auth, u URL) {
Expand Down Expand Up @@ -471,7 +499,16 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m
s.attemptToAddKeyword(authData)

m.FieldType = AUTH_TYPE

if strings.Contains(m.Type, REQUEST_TYPE) {
m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_AUTHORIZATION
} else if strings.Contains(m.Type, FOLDER_TYPE) {
m.LocationType = source_metadatapb.PostmanLocationType_FOLDER_AUTHORIZATION
} else if strings.Contains(m.Type, COLLECTION_TYPE) {
m.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION
}
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, authData)), m)
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, r Request) {
Expand All @@ -484,66 +521,38 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C
KeyValues: r.Header,
}
metadata.Type = originalType + " > header"
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER
s.scanVariableData(ctx, chunksChan, metadata, vars)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

if r.URL.Raw != "" {
metadata.Type = originalType + " > request URL (no query parameters)"
// Note: query parameters are handled separately
u := fmt.Sprintf("%s://%s/%s", r.URL.Protocol, strings.Join(r.URL.Host, "."), strings.Join(r.URL.Path, "/"))
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_URL
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, u)), metadata)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

if len(r.URL.Query) > 0 {
vars := VariableData{
KeyValues: r.URL.Query,
}
metadata.Type = originalType + " > GET parameters (query)"
metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_QUERY_PARAMETER
s.scanVariableData(ctx, chunksChan, metadata, vars)
metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

if r.Auth.Type != "" {
metadata.Type = originalType + " > request auth"
s.scanAuth(ctx, chunksChan, metadata, r.Auth, r.URL)
}

if r.Body.Mode != "" {
metadata.Type = originalType + " > body"
s.scanBody(ctx, chunksChan, metadata, r.Body)
}
}

func (s *Source) scanBody(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, b Body) {
if !m.fromLocal {
m.Link = m.Link + "?tab=body"
}
originalType := m.Type
switch b.Mode {
case "formdata":
m.Type = originalType + " > form data"
vars := VariableData{
KeyValues: b.FormData,
}
s.scanVariableData(ctx, chunksChan, m, vars)
case "urlencoded":
m.Type = originalType + " > url encoded"
vars := VariableData{
KeyValues: b.URLEncoded,
}
s.scanVariableData(ctx, chunksChan, m, vars)
case "raw", "graphql":
data := b.Raw
if b.Mode == "graphql" {
m.Type = originalType + " > graphql"
data = b.GraphQL.Query + " " + b.GraphQL.Variables
}
if b.Mode == "raw" {
m.Type = originalType + " > raw"
}
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, data)), m)
default:
break
}
// We would scan the body, but currently the body has different radio buttons that can be scanned but only the selected one is scanned. The unselected radio button options can still
// have secrets in them but will not be scanned. The selction of the radio button will also change the secret metadata for that particular scanning pass and can create confusion for
// the user as to the status of a secret. We will reimplement at some point.
}

func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, response Response) {
Expand All @@ -558,13 +567,17 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.
KeyValues: response.Header,
}
m.Type = originalType + " > response header"
m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER
s.scanVariableData(ctx, chunksChan, m, vars)
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

// Body in a response is just a string
if response.Body != "" {
m.Type = originalType + " > response body"
m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_BODY
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, response.Body)), m)
m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
}

if response.OriginalRequest.Method != "" {
Expand Down Expand Up @@ -600,14 +613,22 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.
}

m.FieldType = m.Type + " variables"
switch m.FieldType {
case "request > GET parameters (query) variables":
m.Link = m.Link + "?tab=params"
case "request > header variables":
m.Link = m.Link + "?tab=headers"
}
s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(values), m)
}

func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, data string, metadata Metadata) {
if data == "" {
return
}
metadata.FieldType = metadata.Type
if metadata.FieldType == "" {
metadata.FieldType = metadata.Type
}

chunksChan <- &sources.Chunk{
SourceType: s.Type(),
Expand All @@ -630,8 +651,7 @@ func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, d
FolderId: metadata.FolderID,
FolderName: metadata.FolderName,
FieldType: metadata.FieldType,
FieldName: metadata.FieldName,
VariableType: metadata.VarType,
LocationType: metadata.LocationType,
},
},
},
Expand Down
9 changes: 5 additions & 4 deletions pkg/sources/postman/postman_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"io"
"net/http"
"time"

"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
)

const (
Expand Down Expand Up @@ -68,17 +70,16 @@ type Metadata struct {
EnvironmentID string
CollectionInfo Info
FolderID string // UUID of the folder (but not full ID)
FolderName string
FolderName string // Folder path if the item is nested under one or more folders
RequestID string // UUID of the request (but not full ID)
RequestName string
FullID string //full ID of the reference item (created_by + ID) OR just the UUID
Link string //direct link to the folder (could be .json file path)
Type string //folder, request, etc.
EnvironmentName string
VarType string
FieldName string
FieldType string
FieldType string // Path of the item type
fromLocal bool
LocationType source_metadatapb.PostmanLocationType // The distinct Postman location type that the item falls under
}

type Collection struct {
Expand Down
12 changes: 6 additions & 6 deletions pkg/sources/postman/substitution_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,14 @@ func TestSource_BuildSubstituteSet(t *testing.T) {
s := &Source{
sub: NewSubstitution(),
}
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "var1", "value1")
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "var2", "value2")
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "", "value2")
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "continuation_token", "'{{continuation_token}}'") // this caused an infinite loop in the original implementation
s.sub.add(Metadata{Type: GLOBAL_TYPE}, "continuation_token2", "'{{{continuation_token2}}}'") // this caused an infinite loop in the original implementation
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "var1", "value1")
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "var2", "value2")
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "", "value2")
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "continuation_token", "'{{continuation_token}}'") // this caused an infinite loop in the original implementation
s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "continuation_token2", "'{{{continuation_token2}}}'") // this caused an infinite loop in the original implementation

metadata := Metadata{
Type: GLOBAL_TYPE,
Type: ENVIRONMENT_TYPE,
}

testCases := []struct {
Expand Down
29 changes: 25 additions & 4 deletions proto/source_metadata.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -296,10 +296,11 @@ message AzureRepos {
}

message Postman {
reserved 4, 14, 15;
reserved "globals_id", "field_name", "variable_type";
string link = 1;
string workspace_uuid = 2;
string workspace_name = 3;
string globals_id = 4;
string collection_id = 5;
string collection_name = 6;
string environment_id = 7;
Expand All @@ -308,9 +309,29 @@ message Postman {
string request_name = 10;
string folder_id = 11;
string folder_name = 12;
string field_type = 13;
string field_name = 14;
string variable_type = 15;
string field_type = 13; // Do not use this field for transport. It is only used for output to STDOUT.
PostmanLocationType location_type = 16;
}

enum PostmanLocationType {
UNKNOWN_POSTMAN = 0;
REQUEST_QUERY_PARAMETER = 1;
REQUEST_AUTHORIZATION = 2;
REQUEST_HEADER = 3;
REQUEST_BODY_FORM_DATA = 4;
REQUEST_BODY_RAW = 5;
REQUEST_BODY_URL_ENCODED = 6;
REQUEST_BODY_GRAPHQL = 7;
REQUEST_SCRIPT = 8;
REQUEST_URL = 9;
ENVIRONMENT_VARIABLE = 10;
FOLDER_AUTHORIZATION = 11;
FOLDER_SCRIPT = 12;
COLLECTION_SCRIPT = 13;
COLLECTION_VARIABLE = 14;
COLLECTION_AUTHORIZATION = 15;
RESPONSE_BODY = 16;
RESPONSE_HEADER = 17;
}

message Vector {
Expand Down

0 comments on commit 9ecaf07

Please sign in to comment.