[Do Not Merge] PR Experiment

.builders/images/helpers.ps1

@@ -38,3 +38,12 @@ function Add-ToPath() {
     $target="$oldPath;$Append"
     [Environment]::SetEnvironmentVariable("Path", $target, [System.EnvironmentVariableTarget]::User)
 }
+
+
+function RunOnVSConsole() {
+    param(
+        [Parameter(Mandatory = $true)][string] $Command
+    )
+    Write-Host "Running $Command"
+    Start-Process -Wait -NoNewWindow "cmd.exe" -ArgumentList "/c ""$Env:VCVARSALL_BAT"" $Env:DD_TARGET_ARCH && $Command"
+}

.builders/images/linux-aarch64/Dockerfile

@@ -104,11 +104,11 @@ RUN \
  --without-debugger \
  --disable-static
 
-# libpq and pg_config as needed by psycopg2
-RUN \
+# libpq and pg_config as needed by psycopg
+RUN yum install -y flex && \
  DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
- VERSION="16.0" \
- SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
+ VERSION="16.9" \
+ SHA256="07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005" \
  RELATIVE_PATH="postgresql-{{version}}" \
  bash install-from-source.sh --without-readline --with-openssl --without-icu
 # Add paths to pg_config and to the library
@@ -154,8 +154,8 @@ RUN \
 # curl
 RUN \
  DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
- VERSION="8.15.0" \
- SHA256="d85cfc79dc505ff800cb1d321a320183035011fa08cb301356425d86be8fc53c" \
+ VERSION="8.16.0" \
+ SHA256="a21e20476e39eca5a4fc5cfb00acf84bbc1f5d8443ec3853ad14c26b3c85b970" \
  RELATIVE_PATH="curl-{{version}}" \
   bash install-from-source.sh \
     --disable-manual \

.builders/images/linux-aarch64/build_script.sh

@@ -29,7 +29,8 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel
     # That causes the omnibus' health check to flag it. Forcing the build so that we do include it in the wheel.
     always_build+=("pyodbc")
-
+    # Build psycopg to have control over openssl version
+    always_build+=("psycopg-c")
     # We need to build cryptography for FIPS support
     always_build+=("cryptography")
 else

.builders/images/linux-x86_64/Dockerfile

@@ -109,10 +109,10 @@ RUN \
  --disable-static
 
 # libpq and pg_config as needed by psycopg2
-RUN \
+RUN yum install -y flex && \
  DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
- VERSION="16.0" \
- SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
+ VERSION="16.9" \
+ SHA256="07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005" \
  RELATIVE_PATH="postgresql-{{version}}" \
  bash install-from-source.sh --without-readline --with-openssl --without-icu
 # Add paths to pg_config and to the library
@@ -158,8 +158,8 @@ RUN \
 # curl
 RUN \
  DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
- VERSION="8.15.0" \
- SHA256="d85cfc79dc505ff800cb1d321a320183035011fa08cb301356425d86be8fc53c" \
+ VERSION="8.16.0" \
+ SHA256="a21e20476e39eca5a4fc5cfb00acf84bbc1f5d8443ec3853ad14c26b3c85b970" \
  RELATIVE_PATH="curl-{{version}}" \
   bash install-from-source.sh \
     --disable-manual \

.builders/images/linux-x86_64/build_script.sh

@@ -26,7 +26,9 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     # The version of pyodbc is dynamically linked against a version of the odbc which doesn't come included in the wheel
     # That causes the omnibus' health check to flag it. Forcing the build so that we do include it in the wheel.
     always_build+=("pyodbc")
-
+    # Build psycopg to have control over openssl version
+    always_build+=("psycopg-c")
+
     # We need to build cryptography for FIPS support
     always_build+=("cryptography")
 fi

.builders/images/macos/builder_setup.sh

@@ -71,8 +71,8 @@ RELATIVE_PATH="libxslt-{{version}}" \
 
 # curl
 DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
-VERSION="8.15.0" \
-SHA256="d85cfc79dc505ff800cb1d321a320183035011fa08cb301356425d86be8fc53c" \
+VERSION="8.16.0" \
+SHA256="a21e20476e39eca5a4fc5cfb00acf84bbc1f5d8443ec3853ad14c26b3c85b970" \
 RELATIVE_PATH="curl-{{version}}" \
   install-from-source \
     --disable-manual \
@@ -94,6 +94,15 @@ RELATIVE_PATH="curl-{{version}}" \
 # Remove the binary installed so that we consistenly use the same original `curl` binary
 rm "${DD_PREFIX_PATH}/bin/curl"
 
+# libpq and pg_config as needed by psycopg
+DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
+VERSION="16.9" \
+SHA256="07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005" \
+RELATIVE_PATH="postgresql-{{version}}" \
+  install-from-source --without-readline --with-openssl --without-icu
+# Add paths to pg_config and to the library
+echo PATH="${DD_PREFIX_PATH}/bin:${PATH:-}" >> "$DD_ENV_FILE"
+
 # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
 DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
 VERSION="0.9.29" \

.builders/images/windows-x86_64/Dockerfile

@@ -27,6 +27,8 @@ RUN curl -SL --output vs_buildtools.exe https://download.visualstudio.microsoft.
      || IF "%ERRORLEVEL%"=="3010" EXIT 0) `
  && del /q vs_buildtools.exe
 
+ ENV VCVARSALL_BAT="C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvarsall.bat"
+
 # Upgrade PowerShell
 ENV POWERSHELL_VERSION="7.4.0"
 RUN curl -SL --output PowerShell-%POWERSHELL_VERSION%-win-x64.msi https://github.com/PowerShell/PowerShell/releases/download/v%POWERSHELL_VERSION%/PowerShell-%POWERSHELL_VERSION%-win-x64.msi `
@@ -115,9 +117,44 @@ RUN Get-RemoteFile `
     Add-ToPath -Append "C:\perl\perl\bin" && `
     Remove-Item "strawberry-perl-$Env:PERL_VERSION-64bit.zip"
 
-ENV OPENSSL_VERSION="3.4.1"
+# Nasm
+ENV NASM_VERSION="2.16.03"
+RUN Get-RemoteFile `
+      -Uri https://www.nasm.us/pub/nasm/releasebuilds/$Env:NASM_VERSION/win64/nasm-$Env:NASM_VERSION-win64.zip `
+      -Path "nasm-$Env:NASM_VERSION-win64.zip" `
+    -Hash '3ee4782247bcb874378d02f7eab4e294a84d3d15f3f6ee2de2f47a46aa7226e6' && `
+    7z x "nasm-$Env:NASM_VERSION-win64.zip" -o"C:\nasm" && `
+    Add-ToPath -Append "C:\nasm\nasm-$Env:NASM_VERSION" && `
+    Remove-Item "nasm-$Env:NASM_VERSION-win64.zip"
 
-ENV CURL_VERSION="8.15.0"
+ENV OPENSSL_VERSION="3.4.1"
+RUN Get-RemoteFile `
+      -Uri https://www.openssl.org/source/openssl-$Env:OPENSSL_VERSION.tar.gz `
+      -Path openssl-$Env:OPENSSL_VERSION.tar.gz `
+      -Hash '002a2d6b30b58bf4bea46c43bdd96365aaf8daa6c428782aa4feee06da197df3'; `
+    7z x openssl-$Env:OPENSSL_VERSION.tar.gz -r -y && `
+    7z x openssl-$Env:OPENSSL_VERSION.tar -oC:\openssl_3 && `
+    cd C:\openssl_3\openssl-$Env:OPENSSL_VERSION && `
+    RunOnVSConsole -Command `
+    'C:\perl\perl\bin\perl.exe Configure && `
+    nmake && `
+    nmake install_sw'
+# libpq and pg_config as needed by psycopg
+ENV PG_VERSION="16.9"
+COPY pg_config.pl C:\pg_config.pl
+RUN Get-RemoteFile `
+      -Uri https://ftp.postgresql.org/pub/source/v$Env:PG_VERSION/postgresql-$Env:PG_VERSION.tar.bz2 `
+      -Path postgresql-$Env:PG_VERSION.tar.bz2 `
+      -Hash '07c00fb824df0a0c295f249f44691b86e3266753b380c96f633c3311e10bd005'; `
+    7z x postgresql-$Env:PG_VERSION.tar.bz2 -r -y && `
+    7z x postgresql-$Env:PG_VERSION.tar -oC:\postgresql_src && `
+    cd C:\postgresql_src\postgresql-$Env:PG_VERSION\src\tools\msvc && `
+    Copy-Item C:\pg_config.pl -Destination .\config.pl && `
+    RunOnVSConsole -Command 'C:\perl\perl\bin\perl.exe build.pl' && `
+    RunOnVSConsole -Command 'C:\perl\perl\bin\perl.exe install.pl C:\postgresql' && `
+    Add-ToPath -Append "C:\postgresql\bin"
+
+ENV CURL_VERSION="8.16.0"
 
 # Set up runner
 COPY runner_dependencies.txt C:\runner_dependencies.txt

.builders/images/windows-x86_64/build_script.ps1

@@ -24,7 +24,7 @@ Remove-Item "librdkafka-${kafka_version}.tar.gz"
 $triplet = "x64-windows"
 $vcpkg_dir = "C:\vcpkg"
 $librdkafka_dir = "C:\librdkafka\librdkafka-${kafka_version}"
-$desired_commit = "9a023fa7d4c8c9ed3fa5b1be466e605b10b9d220"
+$desired_commit = "7e19f3c64cb636ee21f41bfe8558a6dfaae6236f"
 
 # Clone and configure vcpkg
 if (-Not (Test-Path -Path "$vcpkg_dir\.git")) {

.builders/images/windows-x86_64/pg_config.pl

@@ -0,0 +1,4 @@
+# Overrides over default postgres config
+# https://github.com/postgres/postgres/blob/REL_16_0/src/tools/msvc/config_default.pl
+
+$config->{openssl} = 'c:\Program Files\OpenSSL';

.ddev/config.toml

@@ -158,7 +158,6 @@ exclude = [
     'aerospike',  # v8+ breaks agent build.
     # https://github.com/DataDog/integrations-core/pull/16080
     'lxml',
-    'psycopg', # Pinning psycopg binary to 3.2.7 to align OpenSSL version with the agent until we build wheels ourselves.
     'psutil',
     'pyvmomi', # 9+ has breaking changes
     'pymongo', # Upgrade from 4.8.0 to 4.10.1 causes "AttributeError: module 'pymongo' has no attribute 'mongo_client'"

.deps/image_digests.json

@@ -1,5 +1,5 @@
 {
-  "linux-aarch64": "sha256:84e8c0fe00955018950f5db1ada603f60d33719f10b5277a384a0ec9280d39a2",
-  "linux-x86_64": "sha256:9b4c82ee7c6dad433a77fae04da5e338f4f78a76b62417979d57110c8390f190",
-  "windows-x86_64": "sha256:85f896ccc31d285657ec9392f9e7a9e0d0e6dc0064ee8b125ef40fa4d6c40e79"
+  "linux-aarch64": "sha256:0db1d12a00588fceb11a00d0db5f8e2ae2cd24a8ac94c009ba557c0c2bb83229",
+  "linux-x86_64": "sha256:00eb4cef39bd71cd01edc0d5e5277ad4447cfb1f6776d7ee2d0100688c169070",
+  "windows-x86_64": "sha256:0a18524ff8871647fa4eea4e8989a7a230840ed8a2798d5ac331f82dd926b8e1"
 }

.deps/metadata.json

@@ -1,3 +1,3 @@
 {
-  "sha256": "0095b00faec0ff3c9715f66d499a67fd19bbcf737c364204b4963179ecaf191e"
+  "sha256": "2a6fbeeffedb30afa59ecaf67223497e9da3a7a70fc351db995f0018c39ffc7e"
 }