add huaweicloudlogsreceiver skelethon

internal/huawei/Makefile

@@ -0,0 +1 @@
+include ../../../Makefile.Common

internal/huawei/backoff.go

@@ -0,0 +1,88 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package huawei // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/huawei"
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"go.uber.org/zap"
+)
+
+// Generic function to make an API call with exponential backoff and context cancellation handling.
+func MakeAPICallWithRetry[T any](
+	ctx context.Context,
+	shutdownChan chan struct{},
+	logger *zap.Logger,
+	apiCall func() (*T, error),
+	isThrottlingError func(error) bool,
+	backOffConfig *backoff.ExponentialBackOff,
+) (*T, error) {
+	// Immediately check for context cancellation or server shutdown.
+	select {
+	case <-ctx.Done():
+		return nil, fmt.Errorf("request was cancelled or timed out")
+	case <-shutdownChan:
+		return nil, fmt.Errorf("request is cancelled due to server shutdown")
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	// Make the initial API call.
+	resp, err := apiCall()
+	if err == nil {
+		return resp, nil
+	}
+
+	// If the error is not due to request throttling, return the error.
+	if !isThrottlingError(err) {
+		return nil, err
+	}
+
+	// Initialize the backoff mechanism for retrying the API call.
+	expBackoff := &backoff.ExponentialBackOff{
+		InitialInterval:     backOffConfig.InitialInterval,
+		RandomizationFactor: backOffConfig.RandomizationFactor,
+		Multiplier:          backOffConfig.Multiplier,
+		MaxInterval:         backOffConfig.MaxInterval,
+		MaxElapsedTime:      backOffConfig.MaxElapsedTime,
+		Stop:                backoff.Stop,
+		Clock:               backoff.SystemClock,
+	}
+	expBackoff.Reset()
+	attempts := 0
+
+	// Retry loop for handling throttling errors.
+	for {
+		attempts++
+		delay := expBackoff.NextBackOff()
+		if delay == backoff.Stop {
+			return resp, err
+		}
+		logger.Warn("server busy, retrying request",
+			zap.Int("attempts", attempts),
+			zap.Duration("delay", delay))
+
+		// Handle context cancellation or shutdown before retrying.
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("request was cancelled or timed out")
+		case <-shutdownChan:
+			return nil, fmt.Errorf("request is cancelled due to server shutdown")
+		case <-time.After(delay):
+		}
+
+		// Retry the API call.
+		resp, err = apiCall()
+		if err == nil {
+			return resp, nil
+		}
+		if !isThrottlingError(err) {
+			break
+		}
+	}
+
+	return nil, err
+}

internal/huawei/go.mod

@@ -0,0 +1,16 @@
+module github.com/open-telemetry/opentelemetry-collector-contrib/internal/huawei
+
+go 1.22.3
+
+require (
+	github.com/cenkalti/backoff/v4 v4.3.0
+	github.com/stretchr/testify v1.9.0
+	go.uber.org/zap v1.27.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

receiver/huaweicloudlogsreceiver/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

receiver/huaweicloudlogsreceiver/README.md

@@ -0,0 +1,205 @@
+# Huawei Cloud CES Receiver
+
+<!-- status autogenerated section -->
+| Status        |           |
+| ------------- |-----------|
+| Stability     | [development]: logs   |
+| Distributions | [] |
+| Issues        | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fhuaweicloudlogs%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fhuaweicloudlogs) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fhuaweicloudlogs%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fhuaweicloudlogs) |
+| [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner)    | [@heitorganzeli](https://www.github.com/heitorganzeli), [@narcis96](https://www.github.com/narcis96) |
+
+[development]: https://github.com/open-telemetry/opentelemetry-collector#development
+<!-- end autogenerated section -->
+
+This receiver contains the implementation of the Huawei Cloud [Log Tank Service](https://www.huaweicloud.com/intl/en-us/product/lts.html) (LTS) receiver for the OpenTelemetry Collector. The receiver collects logs from Huawei Cloud's LTS service and sends them to the OpenTelemetry Collector for processing and exporting.
+
+## Configuration
+
+The following settings are required:
+
+- `project_id`: The ID of the project in Huawei Cloud. This is used to identify which project's logs are to be collected. See [Obtaining a Project ID](https://support.huaweicloud.com/intl/en-us/devg-apisign/api-sign-provide-proid.html).
+
+- `region_id`: The ID of the Huawei Cloud region from which logs are collected. For example, `eu-west-101`. The full list of the available regions can be found [here](https://pkg.go.dev/github.com/huaweicloud/huaweicloud-sdk-go-v3@v0.1.110/services/lts/v2/region).
+
+- `log_group_id`: A string indicating the ID of the log group.
+
+- `log_stream_id`: A string indicating the ID of the log stream. See [Obtaining Log Group and Log Stream IDs](https://support.huaweicloud.com/intl/en-us/api-lts/lts_api_0006.html#section1).
+
+- `no_verify_ssl`: A boolean flag indicating whether SSL verification should be disabled. Set to True to disable SSL verification.
+
+- `access_key`: The access key needed for LTS authentification. Check `Huawei Cloud SDK Authentication Setup` section for more details.
+
+- `secret_key`: The secret key needed for LTS authentification. Check `Huawei Cloud SDK Authentication Setup` section for more details.
+
+The following settings are optional:
+
+- `initial_delay`: The delay before the first collection of logs begins. This is a duration field, such as 5s for 5 seconds.
+
+- `collection_interval` (default = `60s`): This is the interval at which this receiver collects logs. This value must be a string readable by Golang's [time.ParseDuration](https://pkg.go.dev/time#ParseDuration). Valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. We recommend a polling interval of at least one minute.
+
+- `retry_on_failure`: The following configurations can be used to control the retry policy of the LTS client. The default values are suitable for most deployment scenarios.
+  - `enabled` (default true)
+  - `initial_interval` (default 100ms)
+  - `max_interval` (default 1s)
+  - `max_elapsed_time` (default 15s)
+  - `randomization_factor` (default 0.5)
+  - `multiplier` (default 1.5)
+
+### Example Configuration
+
+```yaml
+receivers:
+  huaweicloudlogsreceiver:
+    collection_interval: 3h
+    initial_delay: 5s
+    access_key: ${env:HUAWEICLOUD_SDK_AK}
+    secret_key: ${env:HUAWEICLOUD_SDK_SK}
+    project_id: project_1
+    region_id: eu-west-101
+    log_group_id: test-group-id
+    log_stream_id: test-stream-id
+    no_verify_ssl: True
+```
+
+The full list of settings exposed for this receiver are documented [here](./config.go).
+
+### Huawei Cloud SDK Authentication Setup
+
+
+To ensure secure authentication, the Access Key (AK) and Secret Key (SK) used by the Huawei Cloud SDK must be stored in environment variables. See [Obtaining an AK/SK](https://support.huaweicloud.com/intl/en-us/devg-apisign/api-sign-provide-aksk.html).
+
+Before running the application, you need to set the environment variables `HUAWEICLOUD_SDK_AK` and `HUAWEICLOUD_SDK_SK` in your local environment. Here’s how you can do it:
+
+1. Open your terminal.
+2. Set the environment variables by executing the following commands:
+
+    ```sh
+    export HUAWEICLOUD_SDK_AK=your-access-key
+    export HUAWEICLOUD_SDK_SK=your-secret-key
+    ```
+
+3. Verify that the variables are set correctly:
+
+    ```sh
+    echo $HUAWEICLOUD_SDK_AK
+    echo $HUAWEICLOUD_SDK_SK
+    ```
+
+## Error handling
+If you encounter any LTS errors, please refer to the [Huawei Cloud Error Codes](https://support.huaweicloud.com/intl/en-us/ae-ad-1-api-lts/lts_api_0021.html).
+
+## Converting LTS Log Representation to OpenTelemetry Log Representation
+
+| Source Field              | Target Field                                        | Description                                                                                       |
+|---------------------------|----------------------------------------------------|---------------------------------------------------------------------------------------------------|
+| **Log Content (timestamp)** | `logRecord.observedTimestamp`                      | The timestamp extracted from the log content. It follows the layout `"2006-01-02/15:04:05"`. Converted to Unix time in nanoseconds. |
+| **Log Content (body)**    | `logRecord.body`                                    | The main content of the log. Stored as a string in the OTLP log body.                             |
+| **Labels (key)**          | `logRecord.attributes`                              | Each label from the LTS log is converted to an attribute in the OTLP log, where the label's key is used as the attribute's key.   |
+| **Labels (value)**        | `logRecord.attributes`                              | The corresponding value of each LTS log label key is stored as the attribute's value.             |
+| **Line Number**           | `logRecord.attributes.lineNum`                      | The line number from the LTS log, if available. Stored as an attribute with the key `lineNum`.    |
+| **Project ID**            | `resource.attributes.project.id`                    | The project ID used in the configuration file of the receiver..                  |
+| **Region ID**             | `resource.attributes.region.id`                     | The region id used in the configuration file of the receiver.                   |
+| **Group ID**              | `resource.attributes.group.id`                      | The log group id used in the configuration file of the receiver.                    |
+| **Stream ID**             | `resource.attributes.stream.id`                     | The log stream id used in the configuration file of the receiver.                   |
+| *N/A*                     | `resource.attributes.cloud.provider`                | Set to `"huawei_cloud"` as the cloud provider.                                                    |
+| *N/A*                     | `scopeLogs.scope.name`                              | Set to `"huawei_cloud_lts"` as the scope name.                                                    |
+| *N/A*                     | `scopeLogs.scope.version`                           | Set to `"v2"` as the scope version.                                                               |
+
+This mapping ensures that logs from Huawei Cloud's LTS can be seamlessly integrated into systems using the OpenTelemetry Protocol for observability.
+
+
+### Notes
+
+- The `timestamp` field in the source is converted from milliseconds to nanoseconds in the target field.
+- Some fields are added in the target format with constant values to provide additional context and metadata.
+
+### Example:
+
+```json
+[
+    {
+				"content": "2020-07-25/14:40:00 this logis Error NO 2",
+				"line_num": "123",
+				"labels": {
+          "hostIP": "192.168.0.156",
+          "hostId": "9787ef31-f171-4eff-ba71-72d580f11f60",
+          "podName": "default_procname",
+          "clusterId": "CONFIG_FILE",
+          "nameSpace":"CONFIG_FILE",
+          "category": "LTS",
+				}
+    },
+    {
+				"content": "2020-07-25/14:50:00 this logis Error NO 3",
+				"line_num": "456",
+				"labels": {
+          "hostIP": "192.168.0.156",
+          "hostId": "9787ef31-f171-4eff-ba71-72d580f11f60",
+          "podName": "default_procname",
+          "clusterId": "CONFIG_FILE",
+          "nameSpace":"CONFIG_FILE",
+          "category": "LTS",
+				}
+    },
+]
+```
+
+converts to
+
+```json
+{
+  "resourceLogs": [
+    {
+      "resource": {
+        "attributes": {
+            "cloud.provider": "huawei_cloud",
+            "project.id":  "project1",
+            "region.id": "region1",
+            "group.id": "group1",
+            "stream.id":"stream1"
+      },
+      "scopeLogs": [
+        {
+          "scope": {
+            "name": "huawei_cloud_lts",
+            "version": "v2"
+          },
+          "logRecords": [
+            {
+              "observedTimeUnixNano": "1595688000000000000",
+              "body": {
+                "stringValue": "2020-07-25/14:40:00 this log is Error NO 2"
+              },
+              "attributes": {
+                "hostIP": "192.168.0.156",
+                "hostId": "9787ef31-f171-4eff-ba71-72d580f11f60",
+                "podName": "default_procname",
+                "clusterId": "CONFIG_FILE",
+                "nameSpace":"CONFIG_FILE",
+                "category": "LTS",
+                "lineNum": "123"
+              },
+            },
+            {
+              "observedTimeUnixNano": "1595688600000000000",
+              "body": {
+                "stringValue": "2020-07-25/14:50:00 this log is Error NO 3"
+              },
+              "attributes": {
+                  "hostIP": "192.168.0.156",
+                  "hostId": "9787ef31-f171-4eff-ba71-72d580f11f60",
+                  "podName": "default_procname",
+                  "clusterId": "CONFIG_FILE",
+                  "nameSpace":"CONFIG_FILE",
+                  "category": "LTS",
+                   "lineNum": "456"
+              },
+            }
+          ]
+        }
+       ]
+      }
+    }
+  ]
+}
+```