Skip to content

Commit

Permalink
add new method on fixture
Browse files Browse the repository at this point in the history
  • Loading branch information
@khushijain21
khushijain21 committed Feb 14, 2025
1 parent 8c24391 commit d99c931
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 60 deletions.
9 changes: 9 additions & 0 deletions pkg/testing/fixture.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,15 @@ func (f *Fixture) Prepare(ctx context.Context, components ...UsableComponent) er
return nil
}

// Return fixture running directory, should only be called after Prepare()
func (f *Fixture) GetRunningDir(ctx context.Context) (string, error) {
err := f.EnsurePrepared(ctx)
if err != nil {
return "", fmt.Errorf("error preparing binary: %w", err)
}
return f.extractDir, nil
}

// WriteFileToWorkDir sends a file to the working directory alongside the unpacked tar build.
func (f *Fixture) WriteFileToWorkDir(ctx context.Context, data string, name string) error {
err := f.EnsurePrepared(ctx)
Expand Down
152 changes: 92 additions & 60 deletions testing/integration/otel_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ import (
"github.com/stretchr/testify/require"

"github.com/elastic/elastic-agent-libs/mapstr"
"github.com/elastic/elastic-agent-libs/testing/estools"
"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
"github.com/elastic/elastic-agent/pkg/control/v2/client"
aTesting "github.com/elastic/elastic-agent/pkg/testing"
"github.com/elastic/elastic-agent/pkg/testing/define"
"github.com/elastic/elastic-agent/pkg/testing/tools/estools"
"github.com/elastic/elastic-agent/pkg/testing/tools/testcontext"
"github.com/elastic/go-elasticsearch/v8"
)
Expand Down Expand Up @@ -578,7 +578,7 @@ func TestOtelAPMIngestion(t *testing.T) {
"run",
"-e",
"-E", "output.elasticsearch.hosts=['" + esHost + "']",
"-E", "output.elasticsearch.api_key=" + fmt.Sprintf("%s:%s", esApiKey.Id, esApiKey.APIKey),
"-E", "output.elasticsearch.api_key=" + fmt.Sprintf("%s:%s", esApiKey.ID, esApiKey.APIKey),
"-E", "apm-server.host=127.0.0.1:8200",
"-E", "apm-server.ssl.enabled=false",
}
Expand Down Expand Up @@ -1854,28 +1854,53 @@ func TestMonitoringAgentE2E(t *testing.T) {
Stack: &define.Stack{},
})

fbMonitoringIndex := "logs-elastic_agent-default"
// fbMonitoringIndex := "logs-elastic_agent-default"
fbReceiverMonitoringIndex := "logs-otel-default"
logsDrop := filepath.Dir(loggingPath("unit", "darwin"))
tmpDir := t.TempDir()
// endpoint := utils.SocketURLWithFallback(unit, paths.TempDir())

type configOptions struct {
InputPath string
// HttpEndpoint string
HomeDir string
ESEndpoint string
ESApiKey string
BeatsESApiKey string
FBReceiverIndex string
HomeDir string
ESEndpoint string
ESApiKey string
BeatsESApiKey string
}
esEndpoint, err := getESHost()
require.NoError(t, err, "error getting elasticsearch endpoint")
esApiKey, err := createESApiKey(info.ESClient)
require.NoError(t, err, "error creating API key")
require.True(t, len(esApiKey.Encoded) > 1, "api key is invalid %q", esApiKey)

configTemplate := `receivers:
// Now we can actually create the fixture and run it
fixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
require.NoError(t, err)

ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(5*time.Minute))
defer cancel()

err = fixture.Prepare(ctx)
require.NoError(t, err)

// Get the running dir
path, err := fixture.GetRunningDir(ctx)

configTemplate := `
agent.grpc:
port: 6792
outputs:
default:
type: elasticsearch
hosts: {{.ESEndpoint}}
api_key: {{.ESApiKey}}
preset: balanced
agent.monitoring:
enabled: true
logs: true
metrics: false
use_output: default
receivers:
filebeatreceiver/filestream-monitoring-agent:
filebeat:
inputs:
Expand All @@ -1888,8 +1913,8 @@ func TestMonitoringAgentE2E(t *testing.T) {
namespace: default
type: logs
paths:
- {{.InputPath}}/elastic-agent-*.ndjson
- {{.InputPath}}/elastic-agent-watcher-*.ndjson
- {{.InputPath}}/data/elastic-agent-c45b74/logs/elastic-agent-*.ndjson
- {{.InputPath}}/data/elastic-agent-c45b74/logs/elastic-agent-watcher-*.ndjson
close:
on_state_change:
inactive: 5m
Expand Down Expand Up @@ -1990,11 +2015,11 @@ func TestMonitoringAgentE2E(t *testing.T) {
- add_cloud_metadata: ~
- add_docker_metadata: ~
- add_kubernetes_metadata: ~
setup.ilm.enabled: false
setup.template.enabled: false
filebeat.config.modules.enabled: false
logging.event_data.to_stderr: true
logging.event_data.to_files: false
setup.ilm.enabled: false
setup.template.enabled: false
filebeat.config.modules.enabled: false
logging.event_data.to_stderr: true
logging.event_data.to_files: false
exporters:
debug:
use_internal_logger: false
Expand All @@ -2005,7 +2030,7 @@ exporters:
compression: none
api_key: {{.ESApiKey}}
logs_dynamic_index:
enabled: true
enabled: true
batcher:
enabled: true
flush_timeout: 1s
Expand All @@ -2027,31 +2052,20 @@ service:
require.NoError(t,
template.Must(template.New("config").Parse(configTemplate)).Execute(&configBuffer,
configOptions{
InputPath: logsDrop,
InputPath: path,
// HttpEndpoint: endpoint,
HomeDir: tmpDir,
ESEndpoint: esEndpoint,
ESApiKey: esApiKey.Encoded,
BeatsESApiKey: string(beatsApiKey),
FBReceiverIndex: fbReceiverMonitoringIndex,
HomeDir: tmpDir,
ESEndpoint: esEndpoint,
ESApiKey: esApiKey.Encoded,
BeatsESApiKey: string(beatsApiKey),
}))
configContents := configBuffer.Bytes()
fmt.Println(string(configContents))
t.Cleanup(func() {
if t.Failed() {
t.Logf("Contents of agent config file:\n%s\n", string(configContents))
}
})

// Now we can actually create the fixture and run it
fixture, err := define.NewFixtureFromLocalBuild(t, define.Version())
require.NoError(t, err)

ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(5*time.Minute))
defer cancel()

err = fixture.Prepare(ctx)
require.NoError(t, err)
err = fixture.Configure(ctx, configContents)
require.NoError(t, err)

Expand Down Expand Up @@ -2082,8 +2096,20 @@ service:
return true
}, 1*time.Minute, 1*time.Second)

pattern := path + "/data/elastic-agent-c45b74/logs/elastic-agent-*.ndjson"

files, err := filepath.Glob(pattern)
if err != nil {
t.Fatalf("Error checking files: %v", err)
}

if len(files) == 0 {
t.Errorf("No matching files found in path: %s", pattern)
}

var docs estools.Documents
var docs2 estools.Documents
// var docs2 estools.Documents
actualHits := &struct {
Hits int
}{}
Expand All @@ -2092,43 +2118,49 @@ service:
findCtx, findCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer findCancel()

docs, err = estools.GetLogsForIndexWithContext(findCtx, info.ESClient, ".ds-"+fbMonitoringIndex+"*", map[string]interface{}{
"data_stream.dataset": "elastic_agent",
})
// docs, err = estools.GetLogsForIndexWithContext(findCtx, info.ESClient, ".ds-"+fbMonitoringIndex+"*", map[string]interface{}{
// "data_stream.dataset": "elastic_agent",
// })

docs2, err = estools.GetLogsForIndexWithContext(findCtx, info.ESClient, ".ds-"+fbReceiverMonitoringIndex+"*", map[string]interface{}{
"data_stream.dataset": "otel",
})
require.NoError(t, err)

// s, _ := json.MarshalIndent(docs, "", " ")
// fmt.Println(string(s))

// s, _ = json.MarshalIndent(docs2, "", " ")
// fmt.Println(string(s))

actualHits.Hits = docs.Hits.Total.Value

return actualHits.Hits == docs2.Hits.Total.Value
return docs2.Hits.Total.Value > 2
},
1*time.Minute, 1*time.Second, "the number of logs in both index are not same")

doc1 := docs.Hits.Hits[0].Source
doc2 := docs.Hits.Hits[1].Source
ignoredFields := []string{
// Expected to change between filebeat and fbreceiver
"@timestamp",
"agent.ephemeral_id",
"agent.id",

// Missing from fbreceiver doc
"elastic_agent.id",
"elastic_agent.snapshot",
"elastic_agent.version",

// TODO: fbreceiver adds metadata fields that are internal in filebeat.
// Remove this once https://github.com/elastic/beats/pull/42412
// is available in agent.
"@metadata.beat",
"@metadata.type",
"@metadata.version",
}

assertMapsEqual(t, doc1, doc2, ignoredFields, "expected documents to be equal")
// doc1 := docs.Hits.Hits[0].Source
// doc2 := docs.Hits.Hits[1].Source
// ignoredFields := []string{
// // Expected to change between filebeat and fbreceiver
// "@timestamp",
// "agent.ephemeral_id",
// "agent.id",

// // Missing from fbreceiver doc
// "elastic_agent.id",
// "elastic_agent.snapshot",
// "elastic_agent.version",

// // TODO: fbreceiver adds metadata fields that are internal in filebeat.
// // Remove this once https://github.com/elastic/beats/pull/42412
// // is available in agent.
// "@metadata.beat",
// "@metadata.type",
// "@metadata.version",
// }

// assertMapsEqual(t, doc1, doc2, ignoredFields, "expected documents to be equal")
cancel()
cmd.Wait()
}
Expand Down

0 comments on commit d99c931

Please sign in to comment.