Combine esp32 and esp32s3 examples

examples/esp32/.gitignore

@@ -3,3 +3,4 @@ build
 dependencies.lock
 managed_components
 sdkconfig
+sdkconfig.old

examples/esp32/README.md

@@ -21,10 +21,8 @@ $ source export.sh
 
 ### Download
 ```bash
-$ git clone https://github.com/sepfy/libpeer
+$ git clone --recursive https://github.com/sepfy/libpeer
 $ cd libpeer/examples/esp32
-$ idf.py add-dependency "espressif/esp32-camera^2.0.4"
-$ idf.py add-dependency "mdns"
 $ git clone --recursive https://github.com/sepfy/esp_ports.git components/srtp
 ```
 

examples/esp32/components/peer/CMakeLists.txt

@@ -16,7 +16,7 @@ file(GLOB CODES "${PEER_PROJECT_PATH}/src/*.c")
 idf_component_register(
   SRCS ${CODES} ${HTTP_SOURCES} ${MQTT_SOURCES} ${MQTT_SERIALIZER_SOURCES}
   INCLUDE_DIRS "${PEER_PROJECT_PATH}/src" ${HTTP_INCLUDE_PUBLIC_DIRS} ${MQTT_INCLUDE_PUBLIC_DIRS}
-  REQUIRES mbedtls srtp json mdns
+  REQUIRES mbedtls srtp json esp_netif
 )
 
 add_definitions("-DESP32 -DHTTP_DO_NOT_USE_CUSTOM_CONFIG -DMQTT_DO_NOT_USE_CUSTOM_CONFIG")

examples/esp32/main/CMakeLists.txt

@@ -1,5 +1,5 @@
 idf_component_register(SRCS
-  "app_main.c" "camera.c"
+  "app_main.c" "camera.c" "audio.c"
   INCLUDE_DIRS "."
 )
 

examples/esp32/main/app_main.c

@@ -22,12 +22,16 @@
 static const char *TAG = "webrtc";
 
 static TaskHandle_t xPcTaskHandle = NULL;
-static TaskHandle_t xCameraTaskHandle = NULL;
 static TaskHandle_t xPsTaskHandle = NULL;
+static TaskHandle_t xCameraTaskHandle = NULL;
+static TaskHandle_t xAudioTaskHandle = NULL;
 
 extern esp_err_t camera_init();
+extern esp_err_t audio_init();
 extern void camera_task(void *pvParameters);
-extern void wifi_init_sta();
+extern void audio_task(void *pvParameters);
+
+SemaphoreHandle_t xSemaphore = NULL;
 
 PeerConnection *g_pc;
 PeerConnectionState eState = PEER_CONNECTION_CLOSED;
@@ -50,9 +54,9 @@ static void oniceconnectionstatechange(PeerConnectionState state, void *user_dat
   }
 }
 
-static void onmessasge(char *msg, size_t len, void *userdata) {
+static void onmessage(char *msg, size_t len, void *userdata, uint16_t sid) {
 
-  ESP_LOGI(TAG, "Datachannel message: %.*s, size", len, msg);
+  ESP_LOGI(TAG, "Datachannel message: %.*s", len, msg);
 }
 
 void onopen(void *userdata) {
@@ -84,7 +88,10 @@ void peer_connection_task(void *arg) {
 
   for(;;) {
 
-    peer_connection_loop(g_pc);
+    if (xSemaphoreTake(xSemaphore, portMAX_DELAY)) {
+        peer_connection_loop(g_pc);
+        xSemaphoreGive(xSemaphore);
+    }
 
     vTaskDelay(pdMS_TO_TICKS(1));
   }
@@ -99,6 +106,7 @@ void app_main(void) {
    .ice_servers = {
     { .urls = "stun:stun.l.google.com:19302" }
    },
+   .audio_codec = CODEC_PCMA,
    .datachannel = DATA_CHANNEL_BINARY,
   };
 
@@ -124,27 +132,38 @@ void app_main(void) {
     ESP_LOGI(TAG, "Device ID: %s", deviceid);
   }
 
+  xSemaphore = xSemaphoreCreateMutex();
+
   peer_init();
 
   camera_init();
 
+#if defined(CONFIG_ESP32S3_XIAO_SENSE)
+  audio_init();
+#endif
+
   g_pc = peer_connection_create(&config);
   peer_connection_oniceconnectionstatechange(g_pc, oniceconnectionstatechange);
-  peer_connection_ondatachannel(g_pc, onmessasge, onopen, onclose);
+  peer_connection_ondatachannel(g_pc, onmessage, onopen, onclose);
 
   ServiceConfiguration service_config = SERVICE_CONFIG_DEFAULT();
   service_config.client_id = deviceid;
   service_config.pc = g_pc;
+  service_config.mqtt_url = "broker.emqx.io";
   peer_signaling_set_config(&service_config);
   peer_signaling_join_channel();
 
   peer_signaling_join_channel(deviceid, g_pc);
 
-  xTaskCreatePinnedToCore(camera_task, "camera", 4096, NULL, 6, &xCameraTaskHandle, 0);
+#if defined(CONFIG_ESP32S3_XIAO_SENSE)
+  xTaskCreatePinnedToCore(audio_task, "audio", 8192, NULL, 7, &xAudioTaskHandle, 0);
+#endif
+
+  xTaskCreatePinnedToCore(camera_task, "camera", 4096, NULL, 8, &xCameraTaskHandle, 1);
 
-  xTaskCreatePinnedToCore(peer_connection_task, "peer_connection", 8192, NULL, 10, &xPcTaskHandle, 1);
+  xTaskCreatePinnedToCore(peer_connection_task, "peer_connection", 8192, NULL, 5, &xPcTaskHandle, 1);
 
-  xTaskCreatePinnedToCore(peer_signaling_task, "peer_signaling", 8192, NULL, 10, &xPsTaskHandle, 0);
+  xTaskCreatePinnedToCore(peer_signaling_task, "peer_signaling", 8192, NULL, 6, &xPsTaskHandle, 1);
 
   ESP_LOGI(TAG, "[APP] Free memory: %d bytes", esp_get_free_heap_size());
   ESP_LOGI(TAG, "open https://sepfy.github.io/webrtc?deviceId=%s", deviceid);

examples/esp32s3/main/audio.c → examples/esp32/main/audio.c

@@ -3,10 +3,10 @@
 #include "esp_log.h"
 #include "driver/i2s_pdm.h"
 
+#include "esp_audio_enc_default.h"
+#include "esp_audio_enc_reg.h"
+#include "esp_g711_enc.h"
 #include "esp_audio_enc.h"
-#include "esp_audio_enc_def.h"
-#include "esp_audio_def.h"
-#include "esp_opus_enc.h"
 
 #include "peer_connection.h"
 
@@ -24,51 +24,55 @@ i2s_chan_handle_t rx_handle = NULL;
 esp_audio_enc_handle_t enc_handle = NULL;
 esp_audio_enc_in_frame_t aenc_in_frame = { 0 };
 esp_audio_enc_out_frame_t aenc_out_frame = { 0 };
+esp_g711_enc_config_t g711_cfg;
+esp_audio_enc_config_t enc_cfg;
 
 esp_err_t audio_codec_init() {
 
-  uint8_t *inbuf = NULL;
-  uint8_t *outbuf = NULL;
-  int aenc_in_frame_size = 0;
-  int aenc_out_frame_size = 0;
+  uint8_t *read_buf = NULL;
+  uint8_t *write_buf = NULL;
+  int read_size = 0;
+  int out_size = 0;
 
   esp_audio_err_t ret = ESP_AUDIO_ERR_OK;
 
-  esp_opus_enc_config_t config = ESP_OPUS_ENC_CONFIG_DEFAULT();
-  config.sample_rate = ESP_AUDIO_SAMPLE_RATE_8K;
-  config.channel = ESP_AUDIO_MONO;
-  config.bitrate = 18000;
-  config.frame_duration = ESP_OPUS_ENC_FRAME_DURATION_40_MS;
+  esp_audio_enc_register_default();
 
-  ret = esp_opus_enc_open(&config, sizeof(esp_opus_enc_config_t), &enc_handle);
+  g711_cfg.sample_rate = ESP_AUDIO_SAMPLE_RATE_8K;
+  g711_cfg.channel = ESP_AUDIO_MONO;
+  g711_cfg.bits_per_sample = ESP_AUDIO_BIT16;
+
+  enc_cfg.type = ESP_AUDIO_TYPE_G711A;
+  enc_cfg.cfg = &g711_cfg;
+  enc_cfg.cfg_sz = sizeof(g711_cfg);
+
+  ret = esp_audio_enc_open(&enc_cfg, &enc_handle);
   if (ret != ESP_AUDIO_ERR_OK) {
     ESP_LOGE(TAG, "audio encoder open failed");
     return ESP_FAIL;
   }
 
-  ret = esp_opus_enc_get_frame_size(enc_handle, &aenc_in_frame_size, &aenc_out_frame_size);
-  if (ret != ESP_AUDIO_ERR_OK) {
-    ESP_LOGE(TAG, "audio encoder get frame size failed");
-    return ESP_FAIL;
+  int frame_size = (g711_cfg.bits_per_sample * g711_cfg.channel) >> 3;
+  // Get frame_size
+  esp_audio_enc_get_frame_size(enc_handle, &read_size, &out_size);
+  ESP_LOGI(TAG, "audio codec init. frame size: %d, read size: %d, out size: %d", frame_size, read_size, out_size);
+  // 8000HZ duration 20ms
+  if (frame_size == read_size) {
+    read_size *= 8000 / 1000 * 20;
+    out_size *= 8000 / 1000 * 20;
   }
-
-  inbuf = calloc(1, aenc_in_frame_size*2);
-  if (!inbuf) {
-    ESP_LOGE(TAG, "inbuf malloc failed");
+  read_buf = malloc(read_size);
+  write_buf = malloc(out_size);
+  if (read_buf == NULL || write_buf == NULL) {
     return ESP_FAIL;
   }
 
-  outbuf = calloc(1, aenc_out_frame_size);
-  if (!outbuf) {
-    ESP_LOGE(TAG, "outbuf malloc failed");
-    return ESP_FAIL;
-  }
+  aenc_in_frame.buffer = read_buf;
+  aenc_in_frame.len = read_size;
+  aenc_out_frame.buffer = write_buf;
+  aenc_out_frame.len = out_size;
 
-  aenc_in_frame.buffer = inbuf;
-  aenc_in_frame.len = aenc_in_frame_size;
-  aenc_out_frame.buffer = outbuf;
-  aenc_out_frame.len = aenc_out_frame_size;
-  ESP_LOGI(TAG, "audio codec init done. in_frame_size=%d, out_frame_size=%d", aenc_in_frame_size, aenc_out_frame_size);
+  ESP_LOGI(TAG, "audio codec init done. in buffer size: %d, out buffer size: %d", read_size, out_size);
   return 0;
 }
 
@@ -130,7 +134,7 @@ void audio_task(void *arg) {
 
       if (ret == aenc_in_frame.len) {
 
-        if (esp_opus_enc_process(enc_handle, &aenc_in_frame, &aenc_out_frame) == ESP_AUDIO_ERR_OK) {
+        if (esp_audio_enc_process(enc_handle, &aenc_in_frame, &aenc_out_frame) == ESP_AUDIO_ERR_OK) {
 
           peer_connection_send_audio(g_pc, aenc_out_frame.buffer, aenc_out_frame.encoded_bytes);
 

examples/esp32/main/camera.c

@@ -14,6 +14,7 @@
 extern PeerConnection *g_pc;
 extern int gDataChannelOpened;
 extern PeerConnectionState eState;
+extern SemaphoreHandle_t xSemaphore;
 extern int get_timestamp();
 static const char *TAG = "Camera";
 
@@ -134,7 +135,10 @@ void camera_task(void *pvParameters) {
       }
 
       //ESP_LOGI(TAG, "Camera captured. size=%zu, timestamp=%llu", fb->len, fb->timestamp);
-      peer_connection_datachannel_send(g_pc, (char*)fb->buf, fb->len);
+      if (xSemaphoreTake(xSemaphore, portMAX_DELAY)) {
+        peer_connection_datachannel_send(g_pc, (char*)fb->buf, fb->len);
+        xSemaphoreGive(xSemaphore);
+      }
 
       fps++;
 
@@ -148,8 +152,7 @@ void camera_task(void *pvParameters) {
       esp_camera_fb_return(fb);
     }
 
-    // 10 FPS
-    vTaskDelay(pdMS_TO_TICKS(1000/10));
+    vTaskDelay(pdMS_TO_TICKS(1000/20));
   }
 
 }

examples/esp32/main/idf_component.yml

@@ -1,6 +1,6 @@
 ## IDF Component Manager Manifest File
 dependencies:
-  espressif/mdns: "*"
+  espressif/esp_audio_codec: "^2.0.0"
   espressif/esp32-camera: "^2.0.4"
   protocol_examples_common:
     path: ${IDF_PATH}/examples/common_components/protocol_examples_common

examples/esp32/sdkconfig.defaults

@@ -1,23 +1,28 @@
 # This file was generated using idf.py save-defconfig. It can be edited manually.
 # Espressif IoT Development Framework (ESP-IDF) 5.2.2 Project Minimal Configuration
 #
+CONFIG_IDF_TARGET="esp32s3"
 CONFIG_APP_RETRIEVE_LEN_ELF_SHA=16
 CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y
 CONFIG_PARTITION_TABLE_CUSTOM=y
+CONFIG_ESP32S3_XIAO_SENSE=y
+CONFIG_EXAMPLE_WIFI_SSID="myssid"
+CONFIG_EXAMPLE_WIFI_PASSWORD="mypassword"
 CONFIG_EXAMPLE_CONNECT_IPV6=n
 CONFIG_ESP_PHY_REDUCE_TX_POWER=y
 CONFIG_SPIRAM=y
-CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY=y
+CONFIG_SPIRAM_MODE_OCT=y
+CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y
 CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=2048
 CONFIG_ESP_MAIN_TASK_STACK_SIZE=8102
-CONFIG_ESP_INT_WDT_TIMEOUT_MS=300
 CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1=n
 CONFIG_ESP_IPC_TASK_STACK_SIZE=2048
+CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=16
+CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=32
+CONFIG_ESP_WIFI_CACHE_TX_BUFFER_NUM=64
 CONFIG_LWIP_IPV6_AUTOCONFIG=y
 CONFIG_LWIP_IPV6_DHCP6=y
 CONFIG_LWIP_TCP_SND_BUF_DEFAULT=5744
 CONFIG_LWIP_TCP_WND_DEFAULT=5744
 CONFIG_MBEDTLS_EXTERNAL_MEM_ALLOC=y
 CONFIG_MBEDTLS_SSL_PROTO_DTLS=y
-CONFIG_MDNS_TASK_STACK_SIZE=2048
-CONFIG_MDNS_NETWORKING_SOCKET=y